diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 0e24d6132b4cb032d66f505941f1057bf47b527f..74b1e9240d43421dc8d742c2ec2357216b069b48 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -67,3 +67,6 @@ f6d557ee34b6bbdb1dc32f29e34b4a4a8ad35e81 # [libc++] Rename _LIBCPP_INLINE_VISIBILITY to _LIBCPP_HIDE_FROM_ABI 4c198542226223f6a5c5511a1f89b37d15ee10b9 + +# [libc++] Replace uses of _VSTD:: by std:: (#74331) +77a00c0d546cd4aa8311b5b9031ae9ea8cdb050c diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b9a28edc9daf45097d539c0bcc706df89c44e25b..8afd2b9ce4a1c14e02ae36f66af2cd2e59c9d623 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -33,26 +33,42 @@ /lldb/ @JDevlieghere -/mlir/include/mlir/Interfaces/TilingInterface.* @MaheshRavishankar +# Linalg in MLIR. +/mlir/include/mlir/Dialect/Linalg @dcaballe @nicolasvasilache +/mlir/lib/Dialect/Linalg @dcaballe @nicolasvasilache -/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp @MaheshRavishankar -/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp @MaheshRavishankar -/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp @MaheshRavishankar -/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp @MaheshRavishankar -/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp @MaheshRavishankar -/mlir/lib/Interfaces/TilingInterface.* @MaheshRavishankar +# Vector in MLIR. +/mlir/**/*AMX* @dcaballe +/mlir/**/*Neon* @banach-space @dcaballe @nicolasvasilache +/mlir/**/*SME* @banach-space @dcaballe @nicolasvasilache +/mlir/**/*SVE* @banach-space @dcaballe @nicolasvasilache +/mlir/**/*VectorInterfaces* @dcaballe @nicolasvasilache +/mlir/**/*VectorToSCF* @banach-space @dcaballe @nicolasvasilache +/mlir/**/*VectorToLLVM* @banach-space @dcaballe @nicolasvasilache +/mlir/**/*X86Vector* @dcaballe @nicolasvasilache +/mlir/include/mlir/Dialect/Vector @dcaballe @nicolasvasilache +/mlir/lib/Dialect/Vector @dcaballe @nicolasvasilache + +/mlir/include/mlir/Interfaces/TilingInterface.* @MaheshRavishankar @nicolasvasilache + +/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp @MaheshRavishankar @nicolasvasilache +/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp @MaheshRavishankar @nicolasvasilache +/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp @MaheshRavishankar @nicolasvasilache +/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp @MaheshRavishankar @nicolasvasilache +/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp @MaheshRavishankar @nicolasvasilache +/mlir/lib/Interfaces/TilingInterface.* @MaheshRavishankar @nicolasvasilache /mlir/**/*EmulateNarrowType* @hanhanW -/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp @hanhanW -/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp @hanhanW -/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp @hanhanW -/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp @hanhanW -/mlir/lib/Dialect/Tensor/Transforms/FoldIntoPackAndUnpackPatterns.cpp @hanhanW -/mlir/lib/Dialect/Vector/Transforms/* @hanhanW +/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp @hanhanW @nicolasvasilache +/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp @hanhanW @nicolasvasilache +/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp @hanhanW @nicolasvasilache +/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp @hanhanW @nicolasvasilache +/mlir/lib/Dialect/Tensor/Transforms/FoldIntoPackAndUnpackPatterns.cpp @hanhanW @nicolasvasilache +/mlir/lib/Dialect/Vector/Transforms/* @hanhanW @nicolasvasilache # Transform Dialect in MLIR. -/mlir/include/mlir/Dialect/Transform/* @ftynse -/mlir/lib/Dialect/Transform/* @ftynse +/mlir/include/mlir/Dialect/Transform/* @ftynse @nicolasvasilache +/mlir/lib/Dialect/Transform/* @ftynse @nicolasvasilache # SPIR-V in MLIR. /mlir/**/SPIRV/ @antiagainst @kuhar diff --git a/.github/workflows/llvm-project-tests.yml b/.github/workflows/llvm-project-tests.yml index 996cfe41f047f679651b39d29af67b3dd851a1b5..02b1ab75e960eced4a74e9604bf11b0e01e06d04 100644 --- a/.github/workflows/llvm-project-tests.yml +++ b/.github/workflows/llvm-project-tests.yml @@ -10,6 +10,11 @@ on: required: false projects: required: false + extra_cmake_args: + required: false + os_list: + required: false + default: '["ubuntu-latest", "windows-2019", "macOS-11"]' workflow_call: inputs: build_target: @@ -20,6 +25,19 @@ on: required: true type: string + extra_cmake_args: + required: false + type: string + + os_list: + required: false + type: string + # Use windows-2019 due to: + # https://developercommunity.visualstudio.com/t/Prev-Issue---with-__assume-isnan-/1597317 + # We're using a specific version of macOS due to: + # https://github.com/actions/virtual-environments/issues/5900 + default: '["ubuntu-latest", "windows-2019", "macOS-11"]' + concurrency: # Skip intermediate builds: always. # Cancel intermediate builds: only if it is a pull request build. @@ -35,14 +53,7 @@ jobs: strategy: fail-fast: false matrix: - os: - - ubuntu-latest - # Use windows-2019 due to: - # https://developercommunity.visualstudio.com/t/Prev-Issue---with-__assume-isnan-/1597317 - - windows-2019 - # We're using a specific version of macOS due to: - # https://github.com/actions/virtual-environments/issues/5900 - - macOS-11 + os: ${{ fromJSON(inputs.os_list) }} steps: - name: Setup Windows if: startsWith(matrix.os, 'windows') @@ -85,7 +96,7 @@ jobs: # This should be a no-op for non-mac OSes PKG_CONFIG_PATH: /usr/local/Homebrew/Library/Homebrew/os/mac/pkgconfig//12 with: - cmake_args: '-GNinja -DLLVM_ENABLE_PROJECTS="${{ inputs.projects }}" -DCMAKE_BUILD_TYPE=Release -DLLDB_INCLUDE_TESTS=OFF -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache' + cmake_args: '-GNinja -DLLVM_ENABLE_PROJECTS="${{ inputs.projects }}" -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLDB_INCLUDE_TESTS=OFF -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ${{ inputs.extra_cmake_args }}' build_target: '${{ inputs.build_target }}' - name: Build and Test libclc diff --git a/.github/workflows/spirv-tests.yml b/.github/workflows/spirv-tests.yml new file mode 100644 index 0000000000000000000000000000000000000000..0e1d0caca410cfe966351206c8d6b95cb19680f1 --- /dev/null +++ b/.github/workflows/spirv-tests.yml @@ -0,0 +1,29 @@ +name: SPIR-V Tests + +permissions: + contents: read + +on: + workflow_dispatch: + pull_request: + paths: + - 'llvm/lib/Target/SPIRV/**' + - 'llvm/test/CodeGen/SPIRV/**' + - '.github/workflows/spirv-tests.yml' + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + check_spirv: + if: github.repository_owner == 'llvm' + name: Test SPIR-V + uses: ./.github/workflows/llvm-project-tests.yml + with: + build_target: check-llvm-codegen-spirv + projects: + extra_cmake_args: '-DLLVM_TARGETS_TO_BUILD="" -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="SPIRV"' + os_list: '["ubuntu-latest"]' diff --git a/.mailmap b/.mailmap index f4d7ffe3013818c430331c665a77093718bcb8d0..d5464cf9e81529d0fc28e440c16e9ae7980755d5 100644 --- a/.mailmap +++ b/.mailmap @@ -33,6 +33,8 @@ +Jianjian GUAN +Jianjian GUAN Jon Roelofs Jon Roelofs Jonathan Thackray diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h index 182d5ff049c3762d08f95c8c62d69f4ec5ba7784..3a1eae3311bd7623d1263d7627720259e28f83ca 100644 --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -75,6 +75,14 @@ enum IndirectCallPromotionType : char { ICP_ALL /// Perform ICP on calls and jump tables. }; +/// Hash functions supported for BF/BB hashing. +enum class HashFunction : char { + StdHash, /// std::hash, implementation is platform-dependent. Provided for + /// backwards compatibility. + XXH3, /// llvm::xxh3_64bits, the default. + Default = XXH3, +}; + /// Information on a single indirect call to a particular callee. struct IndirectCallProfile { MCSymbol *Symbol; @@ -2234,18 +2242,21 @@ public: /// /// If \p UseDFS is set, process basic blocks in DFS order. Otherwise, use /// the existing layout order. + /// \p HashFunction specifies which function is used for BF hashing. /// /// By default, instruction operands are ignored while calculating the hash. /// The caller can change this via passing \p OperandHashFunc function. /// The return result of this function will be mixed with internal hash. size_t computeHash( - bool UseDFS = false, + bool UseDFS = false, HashFunction HashFunction = HashFunction::Default, OperandHashFuncTy OperandHashFunc = [](const MCOperand &) { return std::string(); }) const; /// Compute hash values for each block of the function. - void computeBlockHashes() const; + /// \p HashFunction specifies which function is used for BB hashing. + void + computeBlockHashes(HashFunction HashFunction = HashFunction::Default) const; void setDWARFUnit(DWARFUnit *Unit) { DwarfUnit = Unit; } diff --git a/bolt/include/bolt/Profile/ProfileYAMLMapping.h b/bolt/include/bolt/Profile/ProfileYAMLMapping.h index 2218a167a74ec09efaa29021cee59341f0596aca..548b528ae2d6534d96cbcea298f9b283c79385f6 100644 --- a/bolt/include/bolt/Profile/ProfileYAMLMapping.h +++ b/bolt/include/bolt/Profile/ProfileYAMLMapping.h @@ -178,6 +178,14 @@ template <> struct ScalarBitSetTraits { } }; +template <> struct ScalarEnumerationTraits { + using HashFunction = llvm::bolt::HashFunction; + static void enumeration(IO &io, HashFunction &value) { + io.enumCase(value, "std-hash", HashFunction::StdHash); + io.enumCase(value, "xxh3", HashFunction::XXH3); + } +}; + namespace bolt { struct BinaryProfileHeader { uint32_t Version{1}; @@ -188,6 +196,7 @@ struct BinaryProfileHeader { std::string Origin; // How the profile was obtained. std::string EventNames; // Events used for sample profile. bool IsDFSOrder{true}; // Whether using DFS block order in function profile + llvm::bolt::HashFunction HashFunction; // Hash used for BB/BF hashing }; } // end namespace bolt @@ -200,6 +209,8 @@ template <> struct MappingTraits { YamlIO.mapOptional("profile-origin", Header.Origin); YamlIO.mapOptional("profile-events", Header.EventNames); YamlIO.mapOptional("dfs-order", Header.IsDFSOrder); + YamlIO.mapOptional("hash-func", Header.HashFunction, + llvm::bolt::HashFunction::StdHash); } }; diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp index be033cf07668801fffc5ed6e8c7e609ba3824ec5..0ac47a53a446775e536ec6b502601169477b07a4 100644 --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -3633,7 +3633,7 @@ BinaryFunction::BasicBlockListType BinaryFunction::dfs() const { return DFS; } -size_t BinaryFunction::computeHash(bool UseDFS, +size_t BinaryFunction::computeHash(bool UseDFS, HashFunction HashFunction, OperandHashFuncTy OperandHashFunc) const { if (size() == 0) return 0; @@ -3652,7 +3652,13 @@ size_t BinaryFunction::computeHash(bool UseDFS, for (const BinaryBasicBlock *BB : Order) HashString.append(hashBlock(BC, *BB, OperandHashFunc)); - return Hash = llvm::xxh3_64bits(HashString); + switch (HashFunction) { + case HashFunction::StdHash: + return Hash = std::hash{}(HashString); + case HashFunction::XXH3: + return Hash = llvm::xxh3_64bits(HashString); + } + llvm_unreachable("Unhandled HashFunction"); } void BinaryFunction::insertBasicBlocks( diff --git a/bolt/lib/Core/BinaryFunctionProfile.cpp b/bolt/lib/Core/BinaryFunctionProfile.cpp index 0d705cd82f5df6c1f23ad29184954e8a102a74c2..55ebe5fc900e65102ccb22a364bbc9262ebce124 100644 --- a/bolt/lib/Core/BinaryFunctionProfile.cpp +++ b/bolt/lib/Core/BinaryFunctionProfile.cpp @@ -225,6 +225,7 @@ void BinaryFunction::mergeProfileDataInto(BinaryFunction &BF) const { for (const BinaryBasicBlock *BBSucc : BB->successors()) { (void)BBSucc; assert(getIndex(BBSucc) == BF.getIndex(*BBMergeSI)); + (void)BBMergeSI; // At this point no branch count should be set to COUNT_NO_PROFILE. assert(BII->Count != BinaryBasicBlock::COUNT_NO_PROFILE && diff --git a/bolt/lib/Core/Exceptions.cpp b/bolt/lib/Core/Exceptions.cpp index 993f3a7770aa8178eb4d25d8e0beac55529dda20..ab1885f6bb5851fb5a9f4d006460dce6570be8a8 100644 --- a/bolt/lib/Core/Exceptions.cpp +++ b/bolt/lib/Core/Exceptions.cpp @@ -108,7 +108,8 @@ void BinaryFunction::parseLSDA(ArrayRef LSDASectionData, DWARFDataExtractor Data( StringRef(reinterpret_cast(LSDASectionData.data()), LSDASectionData.size()), - BC.DwCtx->getDWARFObj().isLittleEndian(), 8); + BC.DwCtx->getDWARFObj().isLittleEndian(), + BC.DwCtx->getDWARFObj().getAddressSize()); uint64_t Offset = getLSDAAddress() - LSDASectionAddress; assert(Data.isValidOffset(Offset) && "wrong LSDA address"); diff --git a/bolt/lib/Passes/IdenticalCodeFolding.cpp b/bolt/lib/Passes/IdenticalCodeFolding.cpp index b4ec89ca8fd79fe51bdecf68b9780b1e77cff8c7..dfbc72e48e5d285b1e38814d0531d0628615b1b0 100644 --- a/bolt/lib/Passes/IdenticalCodeFolding.cpp +++ b/bolt/lib/Passes/IdenticalCodeFolding.cpp @@ -360,9 +360,9 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) { // Pre-compute hash before pushing into hashtable. // Hash instruction operands to minimize hash collisions. - BF.computeHash(opts::ICFUseDFS, [&BC](const MCOperand &Op) { - return hashInstOperand(BC, Op); - }); + BF.computeHash( + opts::ICFUseDFS, HashFunction::Default, + [&BC](const MCOperand &Op) { return hashInstOperand(BC, Op); }); }; ParallelUtilities::PredicateTy SkipFunc = [&](const BinaryFunction &BF) { diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp index a81689bc37469a43877afe948a07ed6049fc2b12..ded0db2cd30b611199c02a2263069d848402e43d 100644 --- a/bolt/lib/Passes/LongJmp.cpp +++ b/bolt/lib/Passes/LongJmp.cpp @@ -202,10 +202,23 @@ LongJmpPass::replaceTargetWithStub(BinaryBasicBlock &BB, MCInst &Inst, } } else if (LocalStubsIter != Stubs.end() && LocalStubsIter->second.count(TgtBB)) { - // If we are replacing a local stub (because it is now out of range), - // use its target instead of creating a stub to jump to another stub + // The TgtBB and TgtSym now are the local out-of-range stub and its label. + // So, we are attempting to restore BB to its previous state without using + // this stub. TgtSym = BC.MIB->getTargetSymbol(*TgtBB->begin()); - TgtBB = BB.getSuccessor(TgtSym, BI); + assert(TgtSym && + "First instruction is expected to contain a target symbol."); + BinaryBasicBlock *TgtBBSucc = TgtBB->getSuccessor(TgtSym, BI); + + // TgtBB might have no successor. e.g. a stub for a function call. + if (TgtBBSucc) { + BB.replaceSuccessor(TgtBB, TgtBBSucc, BI.Count, BI.MispredictedCount); + assert(TgtBB->getExecutionCount() >= BI.Count && + "At least equal or greater than the branch count."); + TgtBB->setExecutionCount(TgtBB->getExecutionCount() - BI.Count); + } + + TgtBB = TgtBBSucc; } BinaryBasicBlock *StubBB = lookupLocalStub(BB, Inst, TgtSym, DotAddress); diff --git a/bolt/lib/Passes/VeneerElimination.cpp b/bolt/lib/Passes/VeneerElimination.cpp index eadbfc17fb9748f3a71a6beeed438c2bcb21b9ae..929c7360b7ffafd36da4cc6561bc5448afa15e6d 100644 --- a/bolt/lib/Passes/VeneerElimination.cpp +++ b/bolt/lib/Passes/VeneerElimination.cpp @@ -89,6 +89,7 @@ void VeneerElimination::runOnFunctions(BinaryContext &BC) { LLVM_DEBUG( dbgs() << "BOLT-INFO: number of linker-inserted veneers call sites: " << VeneerCallers << "\n"); + (void)VeneerCallers; } } // namespace bolt diff --git a/bolt/lib/Profile/StaleProfileMatching.cpp b/bolt/lib/Profile/StaleProfileMatching.cpp index 6fb6f380f71eecd8551d0168641627195bfc179e..26180f1321477972124360473a35f097780b2368 100644 --- a/bolt/lib/Profile/StaleProfileMatching.cpp +++ b/bolt/lib/Profile/StaleProfileMatching.cpp @@ -225,7 +225,7 @@ private: std::unordered_map> OpHashToBlocks; }; -void BinaryFunction::computeBlockHashes() const { +void BinaryFunction::computeBlockHashes(HashFunction HashFunction) const { if (size() == 0) return; @@ -241,12 +241,26 @@ void BinaryFunction::computeBlockHashes() const { // Hashing complete instructions. std::string InstrHashStr = hashBlock( BC, *BB, [&](const MCOperand &Op) { return hashInstOperand(BC, Op); }); - uint64_t InstrHash = llvm::xxh3_64bits(InstrHashStr); - BlendedHashes[I].InstrHash = (uint16_t)InstrHash; + if (HashFunction == HashFunction::StdHash) { + uint64_t InstrHash = std::hash{}(InstrHashStr); + BlendedHashes[I].InstrHash = (uint16_t)hash_value(InstrHash); + } else if (HashFunction == HashFunction::XXH3) { + uint64_t InstrHash = llvm::xxh3_64bits(InstrHashStr); + BlendedHashes[I].InstrHash = (uint16_t)InstrHash; + } else { + llvm_unreachable("Unhandled HashFunction"); + } // Hashing opcodes. std::string OpcodeHashStr = hashBlockLoose(BC, *BB); - OpcodeHashes[I] = llvm::xxh3_64bits(OpcodeHashStr); - BlendedHashes[I].OpcodeHash = (uint16_t)OpcodeHashes[I]; + if (HashFunction == HashFunction::StdHash) { + OpcodeHashes[I] = std::hash{}(OpcodeHashStr); + BlendedHashes[I].OpcodeHash = (uint16_t)hash_value(OpcodeHashes[I]); + } else if (HashFunction == HashFunction::XXH3) { + OpcodeHashes[I] = llvm::xxh3_64bits(OpcodeHashStr); + BlendedHashes[I].OpcodeHash = (uint16_t)OpcodeHashes[I]; + } else { + llvm_unreachable("Unhandled HashFunction"); + } } // Initialize neighbor hash. @@ -258,7 +272,12 @@ void BinaryFunction::computeBlockHashes() const { uint64_t SuccHash = OpcodeHashes[SuccBB->getIndex()]; Hash = hashing::detail::hash_16_bytes(Hash, SuccHash); } - BlendedHashes[I].SuccHash = (uint8_t)Hash; + if (HashFunction == HashFunction::StdHash) { + // Compatibility with old behavior. + BlendedHashes[I].SuccHash = (uint8_t)hash_value(Hash); + } else { + BlendedHashes[I].SuccHash = (uint8_t)Hash; + } // Append hashes of predecessors. Hash = 0; @@ -266,7 +285,12 @@ void BinaryFunction::computeBlockHashes() const { uint64_t PredHash = OpcodeHashes[PredBB->getIndex()]; Hash = hashing::detail::hash_16_bytes(Hash, PredHash); } - BlendedHashes[I].PredHash = (uint8_t)Hash; + if (HashFunction == HashFunction::StdHash) { + // Compatibility with old behavior. + BlendedHashes[I].PredHash = (uint8_t)hash_value(Hash); + } else { + BlendedHashes[I].PredHash = (uint8_t)Hash; + } } // Assign hashes. @@ -682,7 +706,7 @@ bool YAMLProfileReader::inferStaleProfile( << "\"" << BF.getPrintName() << "\"\n"); // Make sure that block hashes are up to date. - BF.computeBlockHashes(); + BF.computeBlockHashes(YamlBP.Header.HashFunction); const BinaryFunction::BasicBlockOrderType BlockOrder( BF.getLayout().block_begin(), BF.getLayout().block_end()); diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp index 079cb352d36e77c9fdac1f79a42322cab1bb3635..ade562ef6fb11162cc52eda683ab3c8179dcfa5c 100644 --- a/bolt/lib/Profile/YAMLProfileReader.cpp +++ b/bolt/lib/Profile/YAMLProfileReader.cpp @@ -83,6 +83,7 @@ bool YAMLProfileReader::parseFunctionProfile( BinaryContext &BC = BF.getBinaryContext(); const bool IsDFSOrder = YamlBP.Header.IsDFSOrder; + const HashFunction HashFunction = YamlBP.Header.HashFunction; bool ProfileMatched = true; uint64_t MismatchedBlocks = 0; uint64_t MismatchedCalls = 0; @@ -98,7 +99,8 @@ bool YAMLProfileReader::parseFunctionProfile( FuncRawBranchCount += YamlSI.Count; BF.setRawBranchCount(FuncRawBranchCount); - if (!opts::IgnoreHash && YamlBF.Hash != BF.computeHash(IsDFSOrder)) { + if (!opts::IgnoreHash && + YamlBF.Hash != BF.computeHash(IsDFSOrder, HashFunction)) { if (opts::Verbosity >= 1) errs() << "BOLT-WARNING: function hash mismatch\n"; ProfileMatched = false; @@ -326,6 +328,17 @@ bool YAMLProfileReader::mayHaveProfileData(const BinaryFunction &BF) { } Error YAMLProfileReader::readProfile(BinaryContext &BC) { + if (opts::Verbosity >= 1) { + outs() << "BOLT-INFO: YAML profile with hash: "; + switch (YamlBP.Header.HashFunction) { + case HashFunction::StdHash: + outs() << "std::hash\n"; + break; + case HashFunction::XXH3: + outs() << "xxh3\n"; + break; + } + } YamlProfileToFunction.resize(YamlBP.Functions.size() + 1); auto profileMatches = [](const yaml::bolt::BinaryFunctionProfile &Profile, @@ -348,7 +361,8 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) { // Recompute hash once per function. if (!opts::IgnoreHash) - Function.computeHash(YamlBP.Header.IsDFSOrder); + Function.computeHash(YamlBP.Header.IsDFSOrder, + YamlBP.Header.HashFunction); if (profileMatches(YamlBF, Function)) matchProfileToFunction(YamlBF, Function); diff --git a/bolt/lib/Profile/YAMLProfileWriter.cpp b/bolt/lib/Profile/YAMLProfileWriter.cpp index 3326d1d8f55965b23f8619d100e725a82666a231..dffd851a1d6f7743bdbf520306b01926e7102384 100644 --- a/bolt/lib/Profile/YAMLProfileWriter.cpp +++ b/bolt/lib/Profile/YAMLProfileWriter.cpp @@ -189,6 +189,7 @@ std::error_code YAMLProfileWriter::writeProfile(const RewriteInstance &RI) { BP.Header.Id = BuildID ? std::string(*BuildID) : ""; BP.Header.Origin = std::string(RI.getProfileReader()->getReaderName()); BP.Header.IsDFSOrder = opts::ProfileUseDFS; + BP.Header.HashFunction = HashFunction::Default; StringSet<> EventNames = RI.getProfileReader()->getEventNames(); if (!EventNames.empty()) { diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index 8cda0b7fcca9f8438708f9ede6fe5e8b6570d6e5..1e8ca569682f712cd4a2ad3cbabda406d8b4f277 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -623,7 +623,9 @@ void RewriteInstance::parseBuildID() { // Reading notes section (see Portable Formats Specification, Version 1.1, // pg 2-5, section "Note Section"). - DataExtractor DE = DataExtractor(Buf, true, 8); + DataExtractor DE = + DataExtractor(Buf, + /*IsLittleEndian=*/true, InputFile->getBytesInAddress()); uint64_t Offset = 0; if (!DE.isValidOffset(Offset)) return; diff --git a/bolt/test/X86/Inputs/blarge_profile_stale.std-hash.yaml b/bolt/test/X86/Inputs/blarge_profile_stale.std-hash.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d520a0d242bf02f6cd1e5cfd379cea0f06df2ee3 --- /dev/null +++ b/bolt/test/X86/Inputs/blarge_profile_stale.std-hash.yaml @@ -0,0 +1,56 @@ +--- +header: + profile-version: 1 + binary-name: 'reader-yaml.test.tmp.exe' + binary-build-id: '' + profile-flags: [ lbr ] + profile-origin: branch profile reader + profile-events: '' + dfs-order: false +functions: + - name: SolveCubic + fid: 6 + hash: 0xC6E9098E973BBE19 + exec: 151 + nblocks: 18 + blocks: + - bid: 0 + insns: 43 + hash: 0xed4db287e71c0000 + exec: 151 + succ: [ { bid: 1, cnt: 151, mis: 2 }, { bid: 7, cnt: 0 } ] + - bid: 1 + insns: 7 + hash: 0x39330000e4560088 + succ: [ { bid: 13, cnt: 151 }, { bid: 2, cnt: 0 } ] + - bid: 13 + insns: 26 + hash: 0xa9700000fe202a7 + succ: [ { bid: 3, cnt: 89 }, { bid: 2, cnt: 10 } ] + - bid: 3 + insns: 9 + hash: 0x62391dad18a700a0 + succ: [ { bid: 5, cnt: 151 } ] + - bid: 5 + insns: 9 + hash: 0x4d906d19ecec0111 + - name: usqrt + fid: 7 + hash: 0x8B62B1F9AD81EA35 + exec: 20 + nblocks: 6 + blocks: + - bid: 0 + insns: 4 + hash: 0x1111111111111111 + exec: 20 + succ: [ { bid: 1, cnt: 0 } ] + - bid: 1 + insns: 9 + hash: 0x27e43a5e10cd0010 + succ: [ { bid: 3, cnt: 320, mis: 171 }, { bid: 2, cnt: 0 } ] + - bid: 3 + insns: 2 + hash: 0x4db935b6471e0039 + succ: [ { bid: 1, cnt: 300, mis: 33 }, { bid: 4, cnt: 20 } ] +... diff --git a/bolt/test/X86/Inputs/blarge_profile_stale.yaml b/bolt/test/X86/Inputs/blarge_profile_stale.yaml index 43b75c99656f1884a4af03c0f07d6d0b7bed2be1..ac46b37b56a121ca6aa3614fab292c83fd714e0a 100644 --- a/bolt/test/X86/Inputs/blarge_profile_stale.yaml +++ b/bolt/test/X86/Inputs/blarge_profile_stale.yaml @@ -7,6 +7,7 @@ header: profile-origin: branch profile reader profile-events: '' dfs-order: false + hash-func: xxh3 functions: - name: SolveCubic fid: 6 diff --git a/bolt/test/X86/reader-stale-yaml-std.test b/bolt/test/X86/reader-stale-yaml-std.test new file mode 100644 index 0000000000000000000000000000000000000000..e0b6ca0645e1954c84258d356dd114985e2ec493 --- /dev/null +++ b/bolt/test/X86/reader-stale-yaml-std.test @@ -0,0 +1,68 @@ +# This script checks that YamlProfileReader in llvm-bolt is reading data +# correctly and stale data is corrected by profile inference. + +RUN: yaml2obj %p/Inputs/blarge.yaml &> %t.exe +RUN: llvm-bolt %t.exe -o %t.null -b %p/Inputs/blarge_profile_stale.std-hash.yaml \ +RUN: --print-cfg --print-only=usqrt,SolveCubic --infer-stale-profile=1 -v=1 \ +RUN: 2>&1 | FileCheck %s + +# Verify that yaml reader works as expected. +CHECK: pre-processing profile using YAML profile reader +CHECK: BOLT-INFO: YAML profile with hash: std::hash + +# Function "SolveCubic" has stale profile, since there is one jump in the +# profile (from bid=13 to bid=2) which is not in the CFG in the binary. The test +# verifies that the inference is able to match two blocks (bid=1 and bid=13) +# using "loose" hashes and then correctly propagate the counts. + +CHECK: Binary Function "SolveCubic" after building cfg { +CHECK: State : CFG constructed +CHECK: Address : 0x400e00 +CHECK: Size : 0x368 +CHECK: Section : .text +CHECK: IsSimple : 1 +CHECK: BB Count : 18 +CHECK: Exec Count : 151 +CHECK: Branch Count: 552 +CHECK: } +# Verify block counts. +CHECK: .LBB00 (43 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB07:]] (mispreds: 0, count: 0), .LFT[[#BB01:]] (mispreds: 0, count: 151) +CHECK: .LFT[[#BB01:]] (5 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB013:]] (mispreds: 0, count: 151), .LFT[[#BB02:]] (mispreds: 0, count: 0) +CHECK: .Ltmp[[#BB03:]] (26 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB05:]] (mispreds: 0, count: 151), .LFT[[#BB04:]] (mispreds: 0, count: 0) +CHECK: .Ltmp[[#BB05:]] (9 instructions, align : 1) +CHECK: .Ltmp[[#BB013:]] (12 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB03:]] (mispreds: 0, count: 151) +CHECK: End of Function "SolveCubic" + +# Function "usqrt" has stale profile, since the number of blocks in the profile +# (nblocks=6) does not match the size of the CFG in the binary. The entry +# block (bid=0) has an incorrect (missing) count, which should be inferred by +# the algorithm. + +CHECK: Binary Function "usqrt" after building cfg { +CHECK: State : CFG constructed +CHECK: Address : 0x401170 +CHECK: Size : 0x43 +CHECK: Section : .text +CHECK: IsSimple : 1 +CHECK: BB Count : 5 +CHECK: Exec Count : 20 +CHECK: Branch Count: 640 +CHECK: } +# Verify block counts. +CHECK: .LBB01 (4 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB113:]] (mispreds: 0, count: 20) +CHECK: .Ltmp[[#BB113:]] (9 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB112:]] (mispreds: 0, count: 320), .LFT[[#BB10:]] (mispreds: 0, count: 0) +CHECK: .LFT[[#BB10:]] (2 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB112:]] (mispreds: 0, count: 0) +CHECK: .Ltmp[[#BB112:]] (2 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB113:]] (mispreds: 0, count: 300), .LFT[[#BB11:]] (mispreds: 0, count: 20) +CHECK: .LFT[[#BB11:]] (2 instructions, align : 1) +CHECK: End of Function "usqrt" +# Check the overall inference stats. +CHECK: 2 out of 7 functions in the binary (28.6%) have non-empty execution profile +CHECK: inferred profile for 2 (100.00% of profiled, 100.00% of stale) functions responsible for {{.*}} samples ({{.*}} out of {{.*}}) diff --git a/clang-tools-extra/clang-doc/Mapper.cpp b/clang-tools-extra/clang-doc/Mapper.cpp index 5264417748a12bd04df57fc46efa2f2de2834541..bb8b7952980ac69e14347441bb108e3c013c3147 100644 --- a/clang-tools-extra/clang-doc/Mapper.cpp +++ b/clang-tools-extra/clang-doc/Mapper.cpp @@ -103,7 +103,7 @@ llvm::SmallString<128> MapASTVisitor::getFile(const NamedDecl *D, .getPresumedLoc(D->getBeginLoc()) .getFilename()); IsFileInRootDir = false; - if (RootDir.empty() || !File.startswith(RootDir)) + if (RootDir.empty() || !File.starts_with(RootDir)) return File; IsFileInRootDir = true; llvm::SmallString<128> Prefix(RootDir); diff --git a/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp b/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp index 52cc2e6569b0520638126a4a72bc403e752c5bcb..0b1e9f59e1a70c7c27edbd005f6747ceb441bea6 100644 --- a/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp +++ b/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp @@ -171,7 +171,7 @@ void ExpandModularHeadersPPCallbacks::InclusionDirective( if (Imported) { serialization::ModuleFile *MF = Compiler.getASTReader()->getModuleManager().lookup( - Imported->getASTFile()); + *Imported->getASTFile()); handleModuleFile(MF); } parseToLocation(DirectiveLoc); diff --git a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp index 6bc9f2dd367dcbd978d9658cc10bad500b9ea78f..05012c7df6a97585762f1777f319bfd1f762aa27 100644 --- a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp @@ -133,6 +133,21 @@ UnusedReturnValueCheck::UnusedReturnValueCheck(llvm::StringRef Name, "::boost::system::error_code"))), AllowCastToVoid(Options.get("AllowCastToVoid", false)) {} +UnusedReturnValueCheck::UnusedReturnValueCheck(llvm::StringRef Name, + ClangTidyContext *Context, + std::string CheckedFunctions) + : UnusedReturnValueCheck(Name, Context, std::move(CheckedFunctions), {}, + false) {} + +UnusedReturnValueCheck::UnusedReturnValueCheck( + llvm::StringRef Name, ClangTidyContext *Context, + std::string CheckedFunctions, std::vector CheckedReturnTypes, + bool AllowCastToVoid) + : ClangTidyCheck(Name, Context), + CheckedFunctions(std::move(CheckedFunctions)), + CheckedReturnTypes(std::move(CheckedReturnTypes)), + AllowCastToVoid(AllowCastToVoid) {} + void UnusedReturnValueCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) { Options.store(Opts, "CheckedFunctions", CheckedFunctions); Options.store(Opts, "CheckedReturnTypes", diff --git a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.h b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.h index b4356f8379fdc85fab5116717a10453a38378429..ab2cc691b894f7b08442d9d3d6410f0b3a70502a 100644 --- a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.h +++ b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.h @@ -31,7 +31,15 @@ public: private: std::string CheckedFunctions; const std::vector CheckedReturnTypes; - const bool AllowCastToVoid; + +protected: + UnusedReturnValueCheck(StringRef Name, ClangTidyContext *Context, + std::string CheckedFunctions); + UnusedReturnValueCheck(StringRef Name, ClangTidyContext *Context, + std::string CheckedFunctions, + std::vector CheckedReturnTypes, + bool AllowCastToVoid); + bool AllowCastToVoid; }; } // namespace clang::tidy::bugprone diff --git a/clang-tools-extra/clang-tidy/hicpp/CMakeLists.txt b/clang-tools-extra/clang-tidy/hicpp/CMakeLists.txt index d12ca275d39647f66038a340e2b4a182fe7e084b..132fbaccccf8a9c59c1effd0cd674a6bc650b6e2 100644 --- a/clang-tools-extra/clang-tidy/hicpp/CMakeLists.txt +++ b/clang-tools-extra/clang-tidy/hicpp/CMakeLists.txt @@ -6,6 +6,7 @@ set(LLVM_LINK_COMPONENTS add_clang_library(clangTidyHICPPModule ExceptionBaseclassCheck.cpp HICPPTidyModule.cpp + IgnoredRemoveResultCheck.cpp MultiwayPathsCoveredCheck.cpp NoAssemblerCheck.cpp SignedBitwiseCheck.cpp diff --git a/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp b/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp index 3749796877120ed0538f8680829291ddda51978d..daa9f398a740ed2624fd9be29f25208b21d5b95d 100644 --- a/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp +++ b/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp @@ -37,6 +37,7 @@ #include "../readability/NamedParameterCheck.h" #include "../readability/UppercaseLiteralSuffixCheck.h" #include "ExceptionBaseclassCheck.h" +#include "IgnoredRemoveResultCheck.h" #include "MultiwayPathsCoveredCheck.h" #include "NoAssemblerCheck.h" #include "SignedBitwiseCheck.h" @@ -57,6 +58,8 @@ public: "hicpp-deprecated-headers"); CheckFactories.registerCheck( "hicpp-exception-baseclass"); + CheckFactories.registerCheck( + "hicpp-ignored-remove-result"); CheckFactories.registerCheck( "hicpp-multiway-paths-covered"); CheckFactories.registerCheck("hicpp-signed-bitwise"); diff --git a/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.cpp b/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3410559d435f6385633d5394dad0ec19c5484212 --- /dev/null +++ b/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.cpp @@ -0,0 +1,28 @@ +//===--- IgnoredRemoveResultCheck.cpp - clang-tidy ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "IgnoredRemoveResultCheck.h" + +namespace clang::tidy::hicpp { + +IgnoredRemoveResultCheck::IgnoredRemoveResultCheck(llvm::StringRef Name, + ClangTidyContext *Context) + : UnusedReturnValueCheck(Name, Context, + "::std::remove;" + "::std::remove_if;" + "::std::unique") { + // The constructor for ClangTidyCheck needs to have been called + // before we can access options via Options.get(). + AllowCastToVoid = Options.get("AllowCastToVoid", true); +} + +void IgnoredRemoveResultCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) { + Options.store(Opts, "AllowCastToVoid", AllowCastToVoid); +} + +} // namespace clang::tidy::hicpp diff --git a/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.h b/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.h new file mode 100644 index 0000000000000000000000000000000000000000..48354c34a8581a49ce5a91d741a3751c4bfcfaf9 --- /dev/null +++ b/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.h @@ -0,0 +1,29 @@ +//===--- IgnoredRemoveResultCheck.h - clang-tidy ----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_HICPP_IGNOREDREMOVERESULTCHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_HICPP_IGNOREDREMOVERESULTCHECK_H + +#include "../bugprone/UnusedReturnValueCheck.h" + +namespace clang::tidy::hicpp { + +/// Ensure that the result of std::remove, std::remove_if and std::unique +/// are not ignored according to rule 17.5.1. +/// +/// For the user-facing documentation see: +/// http://clang.llvm.org/extra/clang-tidy/checks/hicpp/ignored-remove-result.html +class IgnoredRemoveResultCheck : public bugprone::UnusedReturnValueCheck { +public: + IgnoredRemoveResultCheck(StringRef Name, ClangTidyContext *Context); + void storeOptions(ClangTidyOptions::OptionMap &Opts) override; +}; + +} // namespace clang::tidy::hicpp + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_HICPP_IGNOREDREMOVERESULTCHECK_H diff --git a/clang-tools-extra/clang-tidy/misc/IncludeCleanerCheck.cpp b/clang-tools-extra/clang-tidy/misc/IncludeCleanerCheck.cpp index e336ba1ee1fa729af884b48d3be63607f918c506..5ae6caedb7f4c0368c755ae9d9dc89d1b6f8400a 100644 --- a/clang-tools-extra/clang-tidy/misc/IncludeCleanerCheck.cpp +++ b/clang-tools-extra/clang-tidy/misc/IncludeCleanerCheck.cpp @@ -124,7 +124,7 @@ void IncludeCleanerCheck::check(const MatchFinder::MatchResult &Result) { MainFileDecls.push_back(D); } llvm::DenseSet SeenSymbols; - const DirectoryEntry *ResourceDir = + OptionalDirectoryEntryRef ResourceDir = PP->getHeaderSearchInfo().getModuleMap().getBuiltinDir(); // FIXME: Find a way to have less code duplication between include-cleaner // analysis implementation and the below code. diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp index 990e20400fbfcd2beabc009601fc9f2c12b030e4..dfe12c5b6007da5b52640e9c623897e7dd98dce0 100644 --- a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp +++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp @@ -15,6 +15,7 @@ #include "clang/AST/Decl.h" #include "clang/Basic/Diagnostic.h" #include +#include namespace clang::tidy::performance { namespace { @@ -263,19 +264,25 @@ void UnnecessaryCopyInitialization::registerMatchers(MatchFinder *Finder) { void UnnecessaryCopyInitialization::check( const MatchFinder::MatchResult &Result) { - const auto *NewVar = Result.Nodes.getNodeAs("newVarDecl"); + const auto &NewVar = *Result.Nodes.getNodeAs("newVarDecl"); + const auto &BlockStmt = *Result.Nodes.getNodeAs("blockStmt"); + const auto &VarDeclStmt = *Result.Nodes.getNodeAs("declStmt"); + // Do not propose fixes if the DeclStmt has multiple VarDecls or in + // macros since we cannot place them correctly. + const bool IssueFix = + VarDeclStmt.isSingleDecl() && !NewVar.getLocation().isMacroID(); + const bool IsVarUnused = isVariableUnused(NewVar, BlockStmt, *Result.Context); + const bool IsVarOnlyUsedAsConst = + isOnlyUsedAsConst(NewVar, BlockStmt, *Result.Context); + const CheckContext Context{ + NewVar, BlockStmt, VarDeclStmt, *Result.Context, + IssueFix, IsVarUnused, IsVarOnlyUsedAsConst}; const auto *OldVar = Result.Nodes.getNodeAs(OldVarDeclId); const auto *ObjectArg = Result.Nodes.getNodeAs(ObjectArgId); - const auto *BlockStmt = Result.Nodes.getNodeAs("blockStmt"); const auto *CtorCall = Result.Nodes.getNodeAs("ctorCall"); - const auto *Stmt = Result.Nodes.getNodeAs("declStmt"); TraversalKindScope RAII(*Result.Context, TK_AsIs); - // Do not propose fixes if the DeclStmt has multiple VarDecls or in macros - // since we cannot place them correctly. - bool IssueFix = Stmt->isSingleDecl() && !NewVar->getLocation().isMacroID(); - // A constructor that looks like T(const T& t, bool arg = false) counts as a // copy only when it is called with default arguments for the arguments after // the first. @@ -289,74 +296,71 @@ void UnnecessaryCopyInitialization::check( // instantiations where the types differ and rely on implicit conversion would // no longer compile if we switched to a reference. if (differentReplacedTemplateParams( - NewVar->getType(), constructorArgumentType(OldVar, Result.Nodes), + Context.Var.getType(), constructorArgumentType(OldVar, Result.Nodes), *Result.Context)) return; if (OldVar == nullptr) { - handleCopyFromMethodReturn(*NewVar, *BlockStmt, *Stmt, IssueFix, ObjectArg, - *Result.Context); + // `auto NewVar = functionCall();` + handleCopyFromMethodReturn(Context, ObjectArg); } else { - handleCopyFromLocalVar(*NewVar, *OldVar, *BlockStmt, *Stmt, IssueFix, - *Result.Context); + // `auto NewVar = OldVar;` + handleCopyFromLocalVar(Context, *OldVar); } } void UnnecessaryCopyInitialization::handleCopyFromMethodReturn( - const VarDecl &Var, const Stmt &BlockStmt, const DeclStmt &Stmt, - bool IssueFix, const VarDecl *ObjectArg, ASTContext &Context) { - bool IsConstQualified = Var.getType().isConstQualified(); - if (!IsConstQualified && !isOnlyUsedAsConst(Var, BlockStmt, Context)) + const CheckContext &Ctx, const VarDecl *ObjectArg) { + bool IsConstQualified = Ctx.Var.getType().isConstQualified(); + if (!IsConstQualified && !Ctx.IsVarOnlyUsedAsConst) return; if (ObjectArg != nullptr && - !isInitializingVariableImmutable(*ObjectArg, BlockStmt, Context, + !isInitializingVariableImmutable(*ObjectArg, Ctx.BlockStmt, Ctx.ASTCtx, ExcludedContainerTypes)) return; - if (isVariableUnused(Var, BlockStmt, Context)) { - auto Diagnostic = - diag(Var.getLocation(), - "the %select{|const qualified }0variable %1 is copy-constructed " - "from a const reference but is never used; consider " - "removing the statement") - << IsConstQualified << &Var; - if (IssueFix) - recordRemoval(Stmt, Context, Diagnostic); - } else { - auto Diagnostic = - diag(Var.getLocation(), - "the %select{|const qualified }0variable %1 is copy-constructed " - "from a const reference%select{ but is only used as const " - "reference|}0; consider making it a const reference") - << IsConstQualified << &Var; - if (IssueFix) - recordFixes(Var, Context, Diagnostic); - } + diagnoseCopyFromMethodReturn(Ctx); } void UnnecessaryCopyInitialization::handleCopyFromLocalVar( - const VarDecl &NewVar, const VarDecl &OldVar, const Stmt &BlockStmt, - const DeclStmt &Stmt, bool IssueFix, ASTContext &Context) { - if (!isOnlyUsedAsConst(NewVar, BlockStmt, Context) || - !isInitializingVariableImmutable(OldVar, BlockStmt, Context, + const CheckContext &Ctx, const VarDecl &OldVar) { + if (!Ctx.IsVarOnlyUsedAsConst || + !isInitializingVariableImmutable(OldVar, Ctx.BlockStmt, Ctx.ASTCtx, ExcludedContainerTypes)) return; + diagnoseCopyFromLocalVar(Ctx, OldVar); +} - if (isVariableUnused(NewVar, BlockStmt, Context)) { - auto Diagnostic = diag(NewVar.getLocation(), - "local copy %0 of the variable %1 is never modified " - "and never used; " - "consider removing the statement") - << &NewVar << &OldVar; - if (IssueFix) - recordRemoval(Stmt, Context, Diagnostic); - } else { - auto Diagnostic = - diag(NewVar.getLocation(), - "local copy %0 of the variable %1 is never modified; " - "consider avoiding the copy") - << &NewVar << &OldVar; - if (IssueFix) - recordFixes(NewVar, Context, Diagnostic); +void UnnecessaryCopyInitialization::diagnoseCopyFromMethodReturn( + const CheckContext &Ctx) { + auto Diagnostic = + diag(Ctx.Var.getLocation(), + "the %select{|const qualified }0variable %1 is " + "copy-constructed " + "from a const reference%select{%select{ but is only used as const " + "reference|}0| but is never used}2; consider " + "%select{making it a const reference|removing the statement}2") + << Ctx.Var.getType().isConstQualified() << &Ctx.Var << Ctx.IsVarUnused; + maybeIssueFixes(Ctx, Diagnostic); +} + +void UnnecessaryCopyInitialization::diagnoseCopyFromLocalVar( + const CheckContext &Ctx, const VarDecl &OldVar) { + auto Diagnostic = + diag(Ctx.Var.getLocation(), + "local copy %1 of the variable %0 is never modified%select{" + "| and never used}2; consider %select{avoiding the copy|removing " + "the statement}2") + << &OldVar << &Ctx.Var << Ctx.IsVarUnused; + maybeIssueFixes(Ctx, Diagnostic); +} + +void UnnecessaryCopyInitialization::maybeIssueFixes( + const CheckContext &Ctx, DiagnosticBuilder &Diagnostic) { + if (Ctx.IssueFix) { + if (Ctx.IsVarUnused) + recordRemoval(Ctx.VarDeclStmt, Ctx.ASTCtx, Diagnostic); + else + recordFixes(Ctx.Var, Ctx.ASTCtx, Diagnostic); } } diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h index ea009ba9979de97249cd20550d83b73519f13019..ab0f1ecf61063be5b621efcf4a530487d4b82876 100644 --- a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h +++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h @@ -32,14 +32,32 @@ public: void check(const ast_matchers::MatchFinder::MatchResult &Result) override; void storeOptions(ClangTidyOptions::OptionMap &Opts) override; +protected: + // A helper to manipulate the state common to + // `CopyFromMethodReturn` and `CopyFromLocalVar`. + struct CheckContext { + const VarDecl &Var; + const Stmt &BlockStmt; + const DeclStmt &VarDeclStmt; + clang::ASTContext &ASTCtx; + const bool IssueFix; + const bool IsVarUnused; + const bool IsVarOnlyUsedAsConst; + }; + + // Create diagnostics. These are virtual so that derived classes can change + // behaviour. + virtual void diagnoseCopyFromMethodReturn(const CheckContext &Ctx); + virtual void diagnoseCopyFromLocalVar(const CheckContext &Ctx, + const VarDecl &OldVar); + private: - void handleCopyFromMethodReturn(const VarDecl &Var, const Stmt &BlockStmt, - const DeclStmt &Stmt, bool IssueFix, - const VarDecl *ObjectArg, - ASTContext &Context); - void handleCopyFromLocalVar(const VarDecl &NewVar, const VarDecl &OldVar, - const Stmt &BlockStmt, const DeclStmt &Stmt, - bool IssueFix, ASTContext &Context); + void handleCopyFromMethodReturn(const CheckContext &Ctx, + const VarDecl *ObjectArg); + void handleCopyFromLocalVar(const CheckContext &Ctx, const VarDecl &OldVar); + + void maybeIssueFixes(const CheckContext &Ctx, DiagnosticBuilder &Diagnostic); + const std::vector AllowedTypes; const std::vector ExcludedContainerTypes; }; diff --git a/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp b/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp index 831614148c7c25a354b22242e7592aa4f896eb65..759cdd44fd6581c3f2abcdd61b53967f8fd63a61 100644 --- a/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp @@ -20,6 +20,7 @@ #include "clang/Basic/DiagnosticIDs.h" #include "clang/Basic/LLVM.h" #include "clang/Basic/SourceLocation.h" +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" @@ -167,15 +168,13 @@ static const std::array Msgs = {{ // Criteria is a bitset, thus a few helpers are needed. CognitiveComplexity::Criteria operator|(CognitiveComplexity::Criteria LHS, CognitiveComplexity::Criteria RHS) { - return static_cast( - static_cast>(LHS) | - static_cast>(RHS)); + return static_cast(llvm::to_underlying(LHS) | + llvm::to_underlying(RHS)); } CognitiveComplexity::Criteria operator&(CognitiveComplexity::Criteria LHS, CognitiveComplexity::Criteria RHS) { - return static_cast( - static_cast>(LHS) & - static_cast>(RHS)); + return static_cast(llvm::to_underlying(LHS) & + llvm::to_underlying(RHS)); } CognitiveComplexity::Criteria &operator|=(CognitiveComplexity::Criteria &LHS, CognitiveComplexity::Criteria RHS) { diff --git a/clang-tools-extra/clangd/IncludeCleaner.cpp b/clang-tools-extra/clangd/IncludeCleaner.cpp index b0a3c290bad660483327438404f8852783675e06..dda7c9f581f69c7b03517416b2ce2f08cb279d79 100644 --- a/clang-tools-extra/clangd/IncludeCleaner.cpp +++ b/clang-tools-extra/clangd/IncludeCleaner.cpp @@ -397,10 +397,10 @@ IncludeCleanerFindings computeIncludeCleanerFindings(ParsedAST &AST) { std::vector MissingIncludes; llvm::DenseSet Used; trace::Span Tracer("include_cleaner::walkUsed"); - const DirectoryEntry *ResourceDir = AST.getPreprocessor() - .getHeaderSearchInfo() - .getModuleMap() - .getBuiltinDir(); + OptionalDirectoryEntryRef ResourceDir = AST.getPreprocessor() + .getHeaderSearchInfo() + .getModuleMap() + .getBuiltinDir(); include_cleaner::walkUsed( AST.getLocalTopLevelDecls(), /*MacroRefs=*/Macros, AST.getPragmaIncludes().get(), AST.getPreprocessor(), diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp index 49e479abf456210a90b1c596fd786477eb8c0498..37939d36425a970465fa66863e68dce74e13bc02 100644 --- a/clang-tools-extra/clangd/SemanticHighlighting.cpp +++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp @@ -418,7 +418,8 @@ class HighlightingsBuilder { public: HighlightingsBuilder(const ParsedAST &AST, const HighlightingFilter &Filter) : TB(AST.getTokens()), SourceMgr(AST.getSourceManager()), - LangOpts(AST.getLangOpts()), Filter(Filter) {} + LangOpts(AST.getLangOpts()), Filter(Filter), + Resolver(AST.getHeuristicResolver()) {} HighlightingToken &addToken(SourceLocation Loc, HighlightingKind Kind) { auto Range = getRangeForSourceLocation(Loc); @@ -589,7 +590,7 @@ private: HighlightingFilter Filter; std::vector Tokens; std::map> ExtraModifiers; - const HeuristicResolver *Resolver = nullptr; + const HeuristicResolver *Resolver; // returned from addToken(InvalidLoc) HighlightingToken InvalidHighlightingToken; }; diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 9262f9bbfe62a76fa118f9d2a43cee208a3e5329..6d91748e4cef18c7329b2ded0e68125500f44847 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -174,6 +174,12 @@ New checks Flags coroutines that suspend while a lock guard is in scope at the suspension point. +- New :doc:`hicpp-ignored-remove-result + ` check. + + Ensure that the result of ``std::remove``, ``std::remove_if`` and + ``std::unique`` are not ignored according to rule 17.5.1. + - New :doc:`misc-coroutine-hostile-raii ` check. diff --git a/clang-tools-extra/docs/clang-tidy/checks/hicpp/ignored-remove-result.rst b/clang-tools-extra/docs/clang-tidy/checks/hicpp/ignored-remove-result.rst new file mode 100644 index 0000000000000000000000000000000000000000..6ca704ae3e66c71d2b7c1352a7fa164b0195cd70 --- /dev/null +++ b/clang-tools-extra/docs/clang-tidy/checks/hicpp/ignored-remove-result.rst @@ -0,0 +1,24 @@ +.. title:: clang-tidy - hicpp-ignored-remove-result + +hicpp-ignored-remove-result +=========================== + +Ensure that the result of ``std::remove``, ``std::remove_if`` and ``std::unique`` +are not ignored according to +`rule 17.5.1 `_. + +The mutating algorithms ``std::remove``, ``std::remove_if`` and both overloads +of ``std::unique`` operate by swapping or moving elements of the range they are +operating over. On completion, they return an iterator to the last valid +element. In the majority of cases the correct behavior is to use this result as +the first operand in a call to ``std::erase``. + +This check is a subset of :doc:`bugprone-unused-return-value <../bugprone/unused-return-value>` +and depending on used options it can be superfluous to enable both checks. + +Options +------- + +.. option:: AllowCastToVoid + + Controls whether casting return values to ``void`` is permitted. Default: `true`. diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst index e6c02fe48fbf86c465d05b0cf61b9c10743b2068..31f0e090db1d7de7d39a540e50a0131f96b9db91 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/list.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst @@ -226,6 +226,7 @@ Clang-Tidy Checks :doc:`google-runtime-operator `, :doc:`google-upgrade-googletest-case `, "Yes" :doc:`hicpp-exception-baseclass `, + :doc:`hicpp-ignored-remove-result `, :doc:`hicpp-multiway-paths-covered `, :doc:`hicpp-no-assembler `, :doc:`hicpp-signed-bitwise `, diff --git a/clang-tools-extra/include-cleaner/lib/Analysis.cpp b/clang-tools-extra/include-cleaner/lib/Analysis.cpp index 09365c36f9f2c5577eff28385b9fee6f636157ab..450c4c796c141567f8620fe03b631e7d0160b91d 100644 --- a/clang-tools-extra/include-cleaner/lib/Analysis.cpp +++ b/clang-tools-extra/include-cleaner/lib/Analysis.cpp @@ -87,7 +87,7 @@ analyze(llvm::ArrayRef ASTRoots, llvm::StringSet<> Missing; if (!HeaderFilter) HeaderFilter = [](llvm::StringRef) { return false; }; - const DirectoryEntry *ResourceDir = + OptionalDirectoryEntryRef ResourceDir = PP.getHeaderSearchInfo().getModuleMap().getBuiltinDir(); walkUsed(ASTRoots, MacroRefs, PI, PP, [&](const SymbolReference &Ref, llvm::ArrayRef
Providers) { @@ -95,7 +95,7 @@ analyze(llvm::ArrayRef ASTRoots, for (const Header &H : Providers) { if (H.kind() == Header::Physical && (H.physical() == MainFile || - H.physical().getDir() == ResourceDir)) { + (ResourceDir && H.physical().getDir() == *ResourceDir))) { Satisfied = true; } for (const Include *I : Inc.match(H)) { @@ -114,7 +114,7 @@ analyze(llvm::ArrayRef ASTRoots, for (const Include &I : Inc.all()) { if (Used.contains(&I) || !I.Resolved || HeaderFilter(I.Resolved->getFileEntry().tryGetRealPathName()) || - I.Resolved->getFileEntry().getDir() == ResourceDir) + (ResourceDir && I.Resolved->getFileEntry().getDir() == *ResourceDir)) continue; if (PI) { if (PI->shouldKeep(*I.Resolved)) diff --git a/clang-tools-extra/modularize/ModuleAssistant.cpp b/clang-tools-extra/modularize/ModuleAssistant.cpp index 0d4c09987eb1cf1c87f54f57ee955c5cfc6f9b01..5c11ffdb8589d55e224680835d1b887e09b53f55 100644 --- a/clang-tools-extra/modularize/ModuleAssistant.cpp +++ b/clang-tools-extra/modularize/ModuleAssistant.cpp @@ -175,7 +175,7 @@ static bool addModuleDescription(Module *RootModule, llvm::SmallString<256> NativePath, NativePrefix; llvm::sys::path::native(HeaderFilePath, NativePath); llvm::sys::path::native(HeaderPrefix, NativePrefix); - if (NativePath.startswith(NativePrefix)) + if (NativePath.starts_with(NativePrefix)) FilePath = std::string(NativePath.substr(NativePrefix.size() + 1)); else FilePath = std::string(HeaderFilePath); diff --git a/clang-tools-extra/pseudo/include/clang-pseudo/Token.h b/clang-tools-extra/pseudo/include/clang-pseudo/Token.h index 22b72c71cbbabc86ec26031bf3b8048f5c6ee896..859fd7d2b3dfe289e391c1b515ac18554274ac3e 100644 --- a/clang-tools-extra/pseudo/include/clang-pseudo/Token.h +++ b/clang-tools-extra/pseudo/include/clang-pseudo/Token.h @@ -32,6 +32,7 @@ #include "clang/Basic/LangStandard.h" #include "clang/Basic/TokenKinds.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/Support/raw_ostream.h" #include #include @@ -71,10 +72,10 @@ struct Token { Index OriginalIndex = Invalid; // Helpers to get/set Flags based on `enum class`. template bool flag(T Mask) const { - return Flags & uint8_t{static_cast>(Mask)}; + return Flags & uint8_t{llvm::to_underlying(Mask)}; } template void setFlag(T Mask) { - Flags |= uint8_t{static_cast>(Mask)}; + Flags |= uint8_t{llvm::to_underlying(Mask)}; } /// Returns the next token in the stream. this may not be a sentinel. diff --git a/clang-tools-extra/test/clang-tidy/checkers/hicpp/ignored-remove-result.cpp b/clang-tools-extra/test/clang-tidy/checkers/hicpp/ignored-remove-result.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b068f08590989394dc5702f310515d9c4b7f8793 --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/hicpp/ignored-remove-result.cpp @@ -0,0 +1,66 @@ +// RUN: %check_clang_tidy %s hicpp-ignored-remove-result %t +// RUN: %check_clang_tidy -check-suffixes=NOCAST %s hicpp-ignored-remove-result %t -- -config='{CheckOptions: {hicpp-ignored-remove-result.AllowCastToVoid: false}}' + +namespace std { + +template +ForwardIt remove(ForwardIt, ForwardIt, const T &); + +template +ForwardIt remove_if(ForwardIt, ForwardIt, UnaryPredicate); + +template +ForwardIt unique(ForwardIt, ForwardIt); + +template +InputIt find(InputIt, InputIt, const T&); + +class error_code { +}; + +} // namespace std + +std::error_code errorFunc() { + return std::error_code(); +} + +void warning() { + std::remove(nullptr, nullptr, 1); + // CHECK-MESSAGES: [[@LINE-1]]:3: warning: the value returned by this function should not be disregarded; neglecting it may lead to errors + // CHECK-MESSAGES: [[@LINE-2]]:3: note: cast the expression to void to silence this warning + // CHECK-MESSAGES-NOCAST: [[@LINE-3]]:3: warning: the value returned by this function should not be disregarded; neglecting it may lead to errors + + std::remove_if(nullptr, nullptr, nullptr); + // CHECK-MESSAGES: [[@LINE-1]]:3: warning: the value returned by this function should not be disregarded; neglecting it may lead to errors + // CHECK-MESSAGES: [[@LINE-2]]:3: note: cast the expression to void to silence this warning + // CHECK-MESSAGES-NOCAST: [[@LINE-3]]:3: warning: the value returned by this function should not be disregarded; neglecting it may lead to errors + + std::unique(nullptr, nullptr); + // CHECK-MESSAGES: [[@LINE-1]]:3: warning: the value returned by this function should not be disregarded; neglecting it may lead to errors + // CHECK-MESSAGES: [[@LINE-2]]:3: note: cast the expression to void to silence this warning + // CHECK-MESSAGES-NOCAST: [[@LINE-3]]:3: warning: the value returned by this function should not be disregarded; neglecting it may lead to errors +} + +void optionalWarning() { + // No warning unless AllowCastToVoid=false + (void)std::remove(nullptr, nullptr, 1); + // CHECK-MESSAGES-NOCAST: [[@LINE-1]]:9: warning: the value returned by this function should not be disregarded; neglecting it may lead to errors +} + +void noWarning() { + + auto RemoveRetval = std::remove(nullptr, nullptr, 1); + + auto RemoveIfRetval = std::remove_if(nullptr, nullptr, nullptr); + + auto UniqueRetval = std::unique(nullptr, nullptr); + + // Verify that other checks in the baseclass are not used. + // - no warning on std::find since the checker overrides + // bugprone-unused-return-value's checked functions. + std::find(nullptr, nullptr, 1); + // - no warning on return types since the checker disable + // bugprone-unused-return-value's checked return types. + errorFunc(); + (void) errorFunc(); +} diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst index 286405850900aab37177b7af98986d64f8576ce6..d34e867f5e6151cf1bb93d9f71571c5b6433ee68 100644 --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -1483,6 +1483,7 @@ Conditional ``explicit`` __cpp_conditional_explicit C++20 ``using enum`` __cpp_using_enum C++20 C++03 ``if consteval`` __cpp_if_consteval C++23 C++20 ``static operator()`` __cpp_static_call_operator C++23 C++03 +Attributes on Lambda-Expressions C++23 C++11 -------------------------------------- -------------------------------- ------------- ------------- Designated initializers (N494) C99 C89 Array & element qualification (N2607) C23 C89 @@ -3866,6 +3867,30 @@ builtin function, and are named with a ``__opencl_`` prefix. The macros and ``__OPENCL_MEMORY_SCOPE_SUB_GROUP`` are provided, with values corresponding to the enumerators of OpenCL's ``memory_scope`` enumeration.) +__scoped_atomic builtins +------------------------ + +Clang provides a set of atomics taking a memory scope argument. These atomics +are identical to the standard GNU / GCC atomic builtins but taking an extra +memory scope argument. These are designed to be a generic alternative to the +``__opencl_atomic_*`` builtin functions for targets that support atomic memory +scopes. + +Atomic memory scopes are designed to assist optimizations for systems with +several levels of memory hierarchy like GPUs. The following memory scopes are +currently supported: + +* ``__MEMORY_SCOPE_SYSTEM`` +* ``__MEMORY_SCOPE_DEVICE`` +* ``__MEMORY_SCOPE_WRKGRP`` +* ``__MEMORY_SCOPE_WVFRNT`` +* ``__MEMORY_SCOPE_SINGLE`` + +This controls whether or not the atomic operation is ordered with respect to the +whole system, the current device, an OpenCL workgroup, wavefront, or just a +single thread. If these are used on a target that does not support atomic +scopes, then they will behave exactly as the standard GNU atomic builtins. + Low-level ARM exclusive memory builtins --------------------------------------- diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 89ea2f0930ceca1cf3bb6ff4d9ca8960c4d9c491..783dc7333af7e2f096b6df106489c36d615c89e3 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -156,6 +156,9 @@ C++23 Feature Support support for this feature is still experimental, the feature test macro ``__cpp_explicit_this_parameter`` was not set in this version. +- Added a separate warning to warn the use of attributes on lambdas as a C++23 extension + in previous language versions: ``-Wc++23-lambda-attributes``. + C++2c Feature Support ^^^^^^^^^^^^^^^^^^^^^ @@ -201,6 +204,9 @@ C Language Changes number of elements in the flexible array member. This information can improve the results of the array bound sanitizer and the ``__builtin_dynamic_object_size`` builtin. +- Enums will now be represented in TBAA metadata using their actual underlying + integer type. Previously they were treated as chars, which meant they could + alias with all other types. C23 Feature Support ^^^^^^^^^^^^^^^^^^^ @@ -232,6 +238,8 @@ Non-comprehensive list of changes in this release except that it returns the size of a type ignoring tail padding. * ``__builtin_classify_type()`` now classifies ``_BitInt`` values as the return value ``18`` and vector types as return value ``19``, to match GCC 14's behavior. +* The default value of `_MSC_VER` was raised from 1920 to 1933. +* Since MSVC 19.33 added undocumented attribute ``[[msvc::constexpr]]``, this release adds the attribute as well. * Added ``#pragma clang fp reciprocal``. @@ -256,6 +264,16 @@ New Compiler Flags * ``-fopenacc`` was added as a part of the effort to support OpenACC in clang. +* ``-fcx-limited-range`` enables the naive mathematical formulas for complex + division and multiplication with no NaN checking of results. The default is + ``-fno-cx-limited-range``, but this option is enabled by ``-ffast-math``. + +* ``-fcx-fortran-rules`` enables the naive mathematical formulas for complex + multiplication and enables application of Smith's algorithm for complex + division. See SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8 + (1962). The default is ``-fno-cx-fortran-rules``. + + Deprecated Compiler Flags ------------------------- @@ -353,6 +371,8 @@ Improvements to Clang's diagnostics of a base class is not called in the constructor of its derived class. - Clang no longer emits ``-Wmissing-variable-declarations`` for variables declared with the ``register`` storage class. +- Clang's ``-Wswitch-default`` flag now diagnoses whenever a ``switch`` statement + does not have a ``default`` label. - Clang's ``-Wtautological-negation-compare`` flag now diagnoses logical tautologies like ``x && !x`` and ``!x || x`` in expressions. This also makes ``-Winfinite-recursion`` diagnose more cases. @@ -503,6 +523,7 @@ Improvements to Clang's diagnostics 48 | static_assert(1 << 4 == 15); | ~~~~~~~^~~~~ +- Clang now diagnoses definitions of friend function specializations, e.g. ``friend void f<>(int) {}``. Improvements to Clang's time-trace ---------------------------------- @@ -651,6 +672,9 @@ Bug Fixes in This Version - Fixed false positive error emitted by clang when performing qualified name lookup and the current class instantiation has dependent bases. Fixes (`#13826 `_) +- Fix a ``clang-17`` regression where a templated friend with constraints is not + properly applied when its parameters reference an enclosing non-template class. + Fixes (`#71595 `_) - Fix the name of the ifunc symbol emitted for multiversion functions declared with the ``target_clones`` attribute. This addresses a linker error that would otherwise occur when these functions are referenced from other TUs. @@ -658,6 +682,12 @@ Bug Fixes in This Version Fixes (`#64467 `_) - Clang's ``-Wchar-subscripts`` no longer warns on chars whose values are known non-negative constants. Fixes (`#18763 `_) +- Fix crash due to incorrectly allowing conversion functions in copy elision. + Fixes (`#39319 `_) and + (`#60182 `_) and + (`#62157 `_) and + (`#64885 `_) and + (`#65568 `_) Bug Fixes to Compiler Builtins ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -886,11 +916,17 @@ Arm and AArch64 Support - New AArch64 asm constraints have been added for r8-r11(Uci) and r12-r15(Ucj). - Support has been added for the following processors (-mcpu identifiers in parenthesis): +- Support has been added for the following processors (-mcpu identifiers in parenthesis): - * Arm Cortex-A520 (cortex-a520). - * Arm Cortex-A720 (cortex-a720). - * Arm Cortex-X4 (cortex-x4). + For Arm: + + * Cortex-M52 (cortex-m52). + + For AArch64: + + * Cortex-A520 (cortex-a520). + * Cortex-A720 (cortex-a720). + * Cortex-X4 (cortex-x4). Android Support ^^^^^^^^^^^^^^^ @@ -937,6 +973,9 @@ CUDA/HIP Language Changes CUDA Support ^^^^^^^^^^^^ +- Clang now supports CUDA SDK up to 12.3 +- Added support for sm_90a + AIX Support ^^^^^^^^^^^ @@ -976,6 +1015,9 @@ Floating Point Support in Clang ``__builtin_exp10f128`` builtins. - Add ``__builtin_iszero``, ``__builtin_issignaling`` and ``__builtin_issubnormal``. +- Add support for C99's ``#pragma STDC CX_LIMITED_RANGE`` feature. This + enables the naive mathematical formulas for complex multiplication and + division, which are faster but do not correctly handle overflow and infinities. AST Matchers ------------ diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst index 9d64195ee338e2c4657a2db5c43db2b6e6b2c4d2..7c30570437e8b01cd904898f5b57a582e9f4cea1 100644 --- a/clang/docs/UsersManual.rst +++ b/clang/docs/UsersManual.rst @@ -1468,6 +1468,7 @@ floating point semantic models: precise (the default), strict, and fast. With the exception of ``-ffp-contract=fast``, using any of the options below to disable any of the individual optimizations in ``-ffast-math`` will cause ``__FAST_MATH__`` to no longer be set. + ``-ffast-math`` enables ``-fcx-limited-range``. This option implies: @@ -1834,6 +1835,20 @@ floating point semantic models: precise (the default), strict, and fast. * ``16`` - Forces ``_Float16`` operations to be emitted without using excess precision arithmetic. +.. option:: -fcx-limited-range: + + This option enables the naive mathematical formulas for complex division and + multiplication with no NaN checking of results. The default is + ``-fno-cx-limited-range``, but this option is enabled by the ``-ffast-math`` + option. + +.. option:: -fcx-fortran-rules: + + This option enables the naive mathematical formulas for complex + multiplication and enables application of Smith's algorithm for complex + division. See SMITH, R. L. Algorithm 116: Complex division. Commun. + ACM 5, 8 (1962). The default is ``-fno-cx-fortran-rules``. + .. _floating-point-environment: Accessing the floating point environment @@ -3359,8 +3374,8 @@ default for Windows targets. For compatibility with existing code that compiles with MSVC, clang defines the ``_MSC_VER`` and ``_MSC_FULL_VER`` macros. When on Windows, these default to -either the same value as the currently installed version of cl.exe, or ``1920`` -and ``192000000`` (respectively). The ``-fms-compatibility-version=`` flag +either the same value as the currently installed version of cl.exe, or ``1933`` +and ``193300000`` (respectively). The ``-fms-compatibility-version=`` flag overrides these values. It accepts a dotted version tuple, such as 19.00.23506. Changing the MSVC compatibility version makes clang behave more like that version of MSVC. For example, ``-fms-compatibility-version=19`` will enable diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h index a9c4c67a60e8e8e607eb9308075cf298c47afd34..a41f2d66b37b69dcdfdf8206ba34ee4ff5528379 100644 --- a/clang/include/clang/AST/Expr.h +++ b/clang/include/clang/AST/Expr.h @@ -6498,7 +6498,7 @@ public: return cast(SubExprs[ORDER_FAIL]); } Expr *getVal2() const { - if (Op == AO__atomic_exchange) + if (Op == AO__atomic_exchange || Op == AO__scoped_atomic_exchange) return cast(SubExprs[ORDER_FAIL]); assert(NumSubExprs > VAL2); return cast(SubExprs[VAL2]); @@ -6539,7 +6539,9 @@ public: getOp() == AO__opencl_atomic_compare_exchange_weak || getOp() == AO__hip_atomic_compare_exchange_weak || getOp() == AO__atomic_compare_exchange || - getOp() == AO__atomic_compare_exchange_n; + getOp() == AO__atomic_compare_exchange_n || + getOp() == AO__scoped_atomic_compare_exchange || + getOp() == AO__scoped_atomic_compare_exchange_n; } bool isOpenCL() const { @@ -6569,13 +6571,13 @@ public: /// \return empty atomic scope model if the atomic op code does not have /// scope operand. static std::unique_ptr getScopeModel(AtomicOp Op) { - auto Kind = - (Op >= AO__opencl_atomic_load && Op <= AO__opencl_atomic_fetch_max) - ? AtomicScopeModelKind::OpenCL - : (Op >= AO__hip_atomic_load && Op <= AO__hip_atomic_fetch_max) - ? AtomicScopeModelKind::HIP - : AtomicScopeModelKind::None; - return AtomicScopeModel::create(Kind); + if (Op >= AO__opencl_atomic_load && Op <= AO__opencl_atomic_fetch_max) + return AtomicScopeModel::create(AtomicScopeModelKind::OpenCL); + else if (Op >= AO__hip_atomic_load && Op <= AO__hip_atomic_fetch_max) + return AtomicScopeModel::create(AtomicScopeModelKind::HIP); + else if (Op >= AO__scoped_atomic_load && Op <= AO__scoped_atomic_fetch_max) + return AtomicScopeModel::create(AtomicScopeModelKind::Generic); + return AtomicScopeModel::create(AtomicScopeModelKind::None); } /// Get atomic scope model. diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h index 6c147eb8f64062334c969b7c0006a058676184fd..b3ae66e6e769d08cad0f7e8ada1384921132b00d 100644 --- a/clang/include/clang/AST/Type.h +++ b/clang/include/clang/AST/Type.h @@ -36,6 +36,7 @@ #include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/PointerUnion.h" +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/ADT/iterator_range.h" @@ -2384,8 +2385,6 @@ public: bool isRVVType(unsigned ElementCount) const; - bool isRVVType() const; - bool isRVVType(unsigned Bitwidth, bool IsFloat, bool IsBFloat = false) const; /// Return the implicit lifetime for this type, which must not be dependent. @@ -7284,14 +7283,6 @@ inline bool Type::isOpenCLSpecificType() const { isQueueT() || isReserveIDT() || isPipeType() || isOCLExtOpaqueType(); } -inline bool Type::isRVVType() const { -#define RVV_TYPE(Name, Id, SingletonId) \ - isSpecificBuiltinType(BuiltinType::Id) || - return -#include "clang/Basic/RISCVVTypes.def" - false; // end of boolean or operation. -} - inline bool Type::isRVVType(unsigned ElementCount) const { bool Ret = false; #define RVV_VECTOR_TYPE(Name, Id, SingletonId, NumEls, ElBits, NF, IsSigned, \ @@ -7524,7 +7515,7 @@ inline const Type *Type::getPointeeOrArrayElementType() const { /// spaces into a diagnostic with <<. inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &PD, LangAS AS) { - PD.AddTaggedVal(static_cast>(AS), + PD.AddTaggedVal(llvm::to_underlying(AS), DiagnosticsEngine::ArgumentKind::ak_addrspace); return PD; } diff --git a/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def b/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def index ff687a0d178bdea28726a7b0113de267d1cfd9ec..757ee452ced74880adc332e4bfbfdb3e49a3b84e 100644 --- a/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def +++ b/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def @@ -36,6 +36,7 @@ FIXABLE_GADGET(PointerDereference) FIXABLE_GADGET(UPCAddressofArraySubscript) // '&DRE[any]' in an Unspecified Pointer Context FIXABLE_GADGET(UPCStandalonePointer) FIXABLE_GADGET(UPCPreIncrement) // '++Ptr' in an Unspecified Pointer Context +FIXABLE_GADGET(UUCAddAssign) // 'Ptr += n' in an Unspecified Untyped Context FIXABLE_GADGET(PointerAssignment) FIXABLE_GADGET(PointerInit) diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index 121ed203829cec44dcf1ec1eb9fc149ff0712e4e..0d94ea2851c9ab6bc8e2a8057b517ebc7f9e7af8 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -3646,6 +3646,14 @@ def : MutualExclusions<[Owner, Pointer]>; // Microsoft-related attributes +def MSConstexpr : InheritableAttr { + let LangOpts = [MicrosoftExt]; + let Spellings = [CXX11<"msvc", "constexpr">]; + let Subjects = SubjectList<[Function, ReturnStmt], ErrorDiag, + "functions and return statements">; + let Documentation = [MSConstexprDocs]; +} + def MSNoVTable : InheritableAttr, TargetSpecificAttr { let Spellings = [Declspec<"novtable">]; let Subjects = SubjectList<[CXXRecord]>; @@ -4250,7 +4258,8 @@ def HLSLResource : InheritableAttr { "StructuredBuffer", "CBuffer", "Sampler", "TBuffer", "RTAccelerationStructure", "FeedbackTexture2D", "FeedbackTexture2DArray"], - /*opt=*/0, /*fake=*/0, /*isExternalType=*/1> + /*opt=*/0, /*fake=*/0, /*isExternalType=*/1>, + DefaultBoolArgument<"isROV", /*default=*/0> ]; let Documentation = [InternalOnly]; } diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td index bbe4de94cbabef2f225473ad43b0a2c37e184ee3..1a98196834cefc31a73523b79a82035c408db535 100644 --- a/clang/include/clang/Basic/AttrDocs.td +++ b/clang/include/clang/Basic/AttrDocs.td @@ -2659,8 +2659,9 @@ An error will be given if: - Specified values violate subtarget specifications; - Specified values are not compatible with values provided through other attributes; - - The AMDGPU target backend is unable to create machine code that can meet the - request. + +The AMDGPU target backend will emit a warning whenever it is unable to +create machine code that meets the request. }]; } @@ -3656,6 +3657,21 @@ an error: }]; } +def MSConstexprDocs : Documentation { + let Category = DocCatStmt; + let Content = [{ +The ``[[msvc::constexpr]]`` attribute can be applied only to a function +definition or a ``return`` statement. It does not impact function declarations. +A ``[[msvc::constexpr]]`` function cannot be ``constexpr`` or ``consteval``. +A ``[[msvc::constexpr]]`` function is treated as if it were a ``constexpr`` function +when it is evaluated in a constant context of ``[[msvc::constexpr]] return`` statement. +Otherwise, it is treated as a regular function. + +Semantics of this attribute are enabled only under MSVC compatibility +(``-fms-compatibility-version``) 19.33 and later. + }]; +} + def MSNoVTableDocs : Documentation { let Category = DocCatDecl; let Content = [{ diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def index ec39e926889b936c47c89ca21614d0a5d2cf7e99..4dcbaf8a7beaa6598f91cba2d040b36699349364 100644 --- a/clang/include/clang/Basic/Builtins.def +++ b/clang/include/clang/Basic/Builtins.def @@ -904,6 +904,32 @@ BUILTIN(__atomic_signal_fence, "vi", "n") BUILTIN(__atomic_always_lock_free, "bzvCD*", "nE") BUILTIN(__atomic_is_lock_free, "bzvCD*", "nE") +// GNU atomic builtins with atomic scopes. +ATOMIC_BUILTIN(__scoped_atomic_load, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_load_n, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_store, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_store_n, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_exchange, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_exchange_n, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_compare_exchange, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_compare_exchange_n, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_add, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_sub, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_and, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_or, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_xor, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_nand, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_add_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_sub_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_and_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_or_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_xor_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_max_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_min_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_nand_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_min, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_max, "v.", "t") + // OpenCL 2.0 atomic builtins. ATOMIC_BUILTIN(__opencl_atomic_init, "v.", "t") ATOMIC_BUILTIN(__opencl_atomic_load, "v.", "t") diff --git a/clang/include/clang/Basic/BuiltinsNVPTX.def b/clang/include/clang/Basic/BuiltinsNVPTX.def index d74a7d1e55dd28176c55b83b8364bf2dd537ca87..0f2e8260143be78341acd1e3527734709b7caf1e 100644 --- a/clang/include/clang/Basic/BuiltinsNVPTX.def +++ b/clang/include/clang/Basic/BuiltinsNVPTX.def @@ -26,7 +26,9 @@ #pragma push_macro("SM_87") #pragma push_macro("SM_89") #pragma push_macro("SM_90") -#define SM_90 "sm_90" +#pragma push_macro("SM_90a") +#define SM_90a "sm_90a" +#define SM_90 "sm_90|" SM_90a #define SM_89 "sm_89|" SM_90 #define SM_87 "sm_87|" SM_89 #define SM_86 "sm_86|" SM_87 @@ -56,7 +58,11 @@ #pragma push_macro("PTX78") #pragma push_macro("PTX80") #pragma push_macro("PTX81") -#define PTX81 "ptx81" +#pragma push_macro("PTX82") +#pragma push_macro("PTX83") +#define PTX83 "ptx83" +#define PTX82 "ptx82|" PTX83 +#define PTX81 "ptx81|" PTX82 #define PTX80 "ptx80|" PTX81 #define PTX78 "ptx78|" PTX80 #define PTX77 "ptx77|" PTX78 @@ -1055,6 +1061,7 @@ TARGET_BUILTIN(__nvvm_getctarank_shared_cluster, "iv*3", "", AND(SM_90,PTX78)) #pragma pop_macro("SM_87") #pragma pop_macro("SM_89") #pragma pop_macro("SM_90") +#pragma pop_macro("SM_90a") #pragma pop_macro("PTX42") #pragma pop_macro("PTX60") #pragma pop_macro("PTX61") @@ -1072,3 +1079,5 @@ TARGET_BUILTIN(__nvvm_getctarank_shared_cluster, "iv*3", "", AND(SM_90,PTX78)) #pragma pop_macro("PTX78") #pragma pop_macro("PTX80") #pragma pop_macro("PTX81") +#pragma pop_macro("PTX82") +#pragma pop_macro("PTX83") diff --git a/clang/include/clang/Basic/Cuda.h b/clang/include/clang/Basic/Cuda.h index 2d912bdbbd1bc59eda37053faa87be5304803b1f..916cb4b7ef34a7eeb65286c35c5c0f34d51f2091 100644 --- a/clang/include/clang/Basic/Cuda.h +++ b/clang/include/clang/Basic/Cuda.h @@ -39,9 +39,11 @@ enum class CudaVersion { CUDA_118, CUDA_120, CUDA_121, - FULLY_SUPPORTED = CUDA_118, + CUDA_122, + CUDA_123, + FULLY_SUPPORTED = CUDA_123, PARTIALLY_SUPPORTED = - CUDA_121, // Partially supported. Proceed with a warning. + CUDA_123, // Partially supported. Proceed with a warning. NEW = 10000, // Too new. Issue a warning, but allow using it. }; const char *CudaVersionToString(CudaVersion V); @@ -71,6 +73,7 @@ enum class CudaArch { SM_87, SM_89, SM_90, + SM_90a, GFX600, GFX601, GFX602, diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td index 715e0c0dc8fa84e5fa851e11dc1bd4661344d1c6..568000106a84dc75d003089b54383485f9163222 100644 --- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -80,6 +80,7 @@ def remark_fe_backend_optimization_remark_analysis_aliasing : Remark<"%0; " "the '__restrict__' qualifier with the independent array arguments. " "Erroneous results will occur if these options are incorrectly applied!">, BackendInfo, InGroup; + def warn_fe_backend_optimization_failure : Warning<"%0">, BackendInfo, InGroup, DefaultWarn; def note_fe_backend_invalid_loc : Note<"could " diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td index ff028bbbf74261e0829ca0294e27acaaf2ac9e80..caee2dc6daadb606dde10e47d2d5404081526d80 100644 --- a/clang/include/clang/Basic/DiagnosticGroups.td +++ b/clang/include/clang/Basic/DiagnosticGroups.td @@ -632,7 +632,7 @@ def ShadowAll : DiagGroup<"shadow-all", [Shadow, ShadowFieldInConstructor, def Shorten64To32 : DiagGroup<"shorten-64-to-32">; def : DiagGroup<"sign-promo">; def SignCompare : DiagGroup<"sign-compare">; -def : DiagGroup<"switch-default">; +def SwitchDefault : DiagGroup<"switch-default">; def : DiagGroup<"synth">; def SizeofArrayArgument : DiagGroup<"sizeof-array-argument">; def SizeofArrayDecay : DiagGroup<"sizeof-array-decay">; @@ -1126,6 +1126,8 @@ def FutureAttrs : DiagGroup<"future-attribute-extensions", [CXX14Attrs, CXX17Attrs, CXX20Attrs]>; +def CXX23AttrsOnLambda : DiagGroup<"c++23-lambda-attributes">; + // A warning group for warnings about using C++11 features as extensions in // earlier C++ versions. def CXX11 : DiagGroup<"c++11-extensions", [CXX11ExtraSemi, CXX11InlineNamespace, @@ -1145,7 +1147,7 @@ def CXX20 : DiagGroup<"c++20-extensions", [CXX20Designator, CXX20Attrs]>; // A warning group for warnings about using C++23 features as extensions in // earlier C++ versions. -def CXX23 : DiagGroup<"c++23-extensions">; +def CXX23 : DiagGroup<"c++23-extensions", [CXX23AttrsOnLambda]>; // A warning group for warnings about using C++26 features as extensions in // earlier C++ versions. diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td index e0e199e0d7300392537c72d88669ce64a0d2439a..6150fc36430ab16111148b25d725d69955e21842 100644 --- a/clang/include/clang/Basic/DiagnosticParseKinds.td +++ b/clang/include/clang/Basic/DiagnosticParseKinds.td @@ -1035,7 +1035,7 @@ def err_capture_default_first : Error< "capture default must be first">; def ext_decl_attrs_on_lambda : ExtWarn< "%select{an attribute specifier sequence|%0}1 in this position " - "is a C++23 extension">, InGroup; + "is a C++23 extension">, InGroup; def ext_lambda_missing_parens : ExtWarn< "lambda without a parameter clause is a C++23 extension">, InGroup; diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 6dfb2d7195203a3d900674331f036dd8306ab224..94e97a891baedcdcbbfb02805d771201d82d8db4 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -1669,6 +1669,8 @@ def err_qualified_friend_def : Error< "friend function definition cannot be qualified with '%0'">; def err_friend_def_in_local_class : Error< "friend function cannot be defined in a local class">; +def err_friend_specialization_def : Error< + "friend function specialization cannot be defined">; def err_friend_not_first_in_declaration : Error< "'friend' must appear first in a non-function declaration">; def err_using_decl_friend : Error< @@ -2884,6 +2886,8 @@ def warn_cxx11_compat_constexpr_body_multiple_return : Warning< InGroup, DefaultIgnore; def note_constexpr_body_previous_return : Note< "previous return statement is here">; +def err_ms_constexpr_cannot_be_applied : Error< + "attribute 'msvc::constexpr' cannot be applied to the %select{constexpr|consteval|virtual}0 function %1">; // C++20 function try blocks in constexpr def ext_constexpr_function_try_block_cxx20 : ExtWarn< @@ -4933,6 +4937,8 @@ def err_template_param_shadow : Error< def ext_template_param_shadow : ExtWarn< err_template_param_shadow.Summary>, InGroup; def note_template_param_here : Note<"template parameter is declared here">; +def note_template_param_external : Note< + "template parameter from hidden source: %0">; def warn_template_export_unsupported : Warning< "exported templates are unsupported">; def err_template_outside_namespace_or_class_scope : Error< @@ -5061,6 +5067,8 @@ def err_template_arg_list_different_arity : Error< "%select{class template|function template|variable template|alias template|" "template template parameter|concept|template}1 %2">; def note_template_decl_here : Note<"template is declared here">; +def note_template_decl_external : Note< + "template declaration from hidden source: %0">; def err_template_arg_must_be_type : Error< "template argument for template type parameter must be a type">; def err_template_arg_must_be_type_suggest : Error< @@ -10060,6 +10068,8 @@ def warn_missing_case : Warning<"%plural{" "3:enumeration values %1, %2, and %3 not handled in switch|" ":%0 enumeration values not handled in switch: %1, %2, %3...}0">, InGroup; +def warn_switch_default : Warning<"'switch' missing 'default' label">, + InGroup, DefaultIgnore; def warn_unannotated_fallthrough : Warning< "unannotated fall-through between switch labels">, @@ -11992,7 +12002,7 @@ def warn_tcb_enforcement_violation : Warning< // RISC-V builtin required extension warning def err_riscv_builtin_requires_extension : Error< - "builtin requires%select{| at least one of the following extensions to be enabled}0: %1">; + "builtin requires%select{| at least one of the following extensions}0: %1">; def err_riscv_builtin_invalid_lmul : Error< "LMUL argument must be in the range [0,3] or [5,7]">; def err_riscv_type_requires_extension : Error< diff --git a/clang/include/clang/Basic/DirectoryEntry.h b/clang/include/clang/Basic/DirectoryEntry.h index 5d083e68facd7a6b73975bab00dc29e8984e8ae5..906c2e9af23b3135419ad6658a51baadf1d07763 100644 --- a/clang/include/clang/Basic/DirectoryEntry.h +++ b/clang/include/clang/Basic/DirectoryEntry.h @@ -245,78 +245,4 @@ template <> struct DenseMapInfo { } // end namespace llvm -namespace clang { - -/// Wrapper around OptionalDirectoryEntryRef that degrades to 'const -/// DirectoryEntry*', facilitating incremental patches to propagate -/// DirectoryEntryRef. -/// -/// This class can be used as return value or field where it's convenient for -/// an OptionalDirectoryEntryRef to degrade to a 'const DirectoryEntry*'. The -/// purpose is to avoid code churn due to dances like the following: -/// \code -/// // Old code. -/// lvalue = rvalue; -/// -/// // Temporary code from an incremental patch. -/// OptionalDirectoryEntryRef MaybeF = rvalue; -/// lvalue = MaybeF ? &MaybeF.getDirectoryEntry() : nullptr; -/// -/// // Final code. -/// lvalue = rvalue; -/// \endcode -/// -/// FIXME: Once DirectoryEntryRef is "everywhere" and DirectoryEntry::LastRef -/// and DirectoryEntry::getName have been deleted, delete this class and -/// replace instances with OptionalDirectoryEntryRef. -class OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr - : public OptionalDirectoryEntryRef { -public: - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr() = default; - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr( - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &&) = default; - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr( - const OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &) = default; - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr & - operator=(OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &&) = default; - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr & - operator=(const OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &) = default; - - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr(std::nullopt_t) {} - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr(DirectoryEntryRef Ref) - : OptionalDirectoryEntryRef(Ref) {} - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr( - OptionalDirectoryEntryRef MaybeRef) - : OptionalDirectoryEntryRef(MaybeRef) {} - - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr & - operator=(std::nullopt_t) { - OptionalDirectoryEntryRef::operator=(std::nullopt); - return *this; - } - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &operator=(DirectoryEntryRef Ref) { - OptionalDirectoryEntryRef::operator=(Ref); - return *this; - } - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr & - operator=(OptionalDirectoryEntryRef MaybeRef) { - OptionalDirectoryEntryRef::operator=(MaybeRef); - return *this; - } - - /// Degrade to 'const DirectoryEntry *' to allow DirectoryEntry::LastRef and - /// DirectoryEntry::getName have been deleted, delete this class and replace - /// instances with OptionalDirectoryEntryRef - operator const DirectoryEntry *() const { - return has_value() ? &(*this)->getDirEntry() : nullptr; - } -}; - -static_assert(std::is_trivially_copyable< - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr>::value, - "OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr should be " - "trivially copyable"); - -} // end namespace clang - #endif // LLVM_CLANG_BASIC_DIRECTORYENTRY_H diff --git a/clang/include/clang/Basic/FPOptions.def b/clang/include/clang/Basic/FPOptions.def index 5b923a1944e509ad31736bf48d6932364bf2c797..79f04c89c9fedc14eaca6928e6677f9db260bb8d 100644 --- a/clang/include/clang/Basic/FPOptions.def +++ b/clang/include/clang/Basic/FPOptions.def @@ -28,4 +28,5 @@ OPTION(FPEvalMethod, LangOptions::FPEvalMethodKind, 2, AllowApproxFunc) OPTION(Float16ExcessPrecision, LangOptions::ExcessPrecisionKind, 2, FPEvalMethod) OPTION(BFloat16ExcessPrecision, LangOptions::ExcessPrecisionKind, 2, Float16ExcessPrecision) OPTION(MathErrno, bool, 1, BFloat16ExcessPrecision) +OPTION(ComplexRange, LangOptions::ComplexRangeKind, 2, MathErrno) #undef OPTION diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def index adaf2e413f2f6deaf118197e96d1b2fb58ec7a4b..7473e00a7bd86bae6f92d4882070f2337a169c65 100644 --- a/clang/include/clang/Basic/Features.def +++ b/clang/include/clang/Basic/Features.def @@ -89,6 +89,8 @@ FEATURE(blocks, LangOpts.Blocks) FEATURE(c_thread_safety_attributes, true) FEATURE(cxx_exceptions, LangOpts.CXXExceptions) FEATURE(cxx_rtti, LangOpts.RTTI &&LangOpts.RTTIData) +EXTENSION(define_target_os_macros, + PP.getPreprocessorOpts().DefineTargetOSMacros) FEATURE(enumerator_attributes, true) FEATURE(nullability, true) FEATURE(nullability_on_arrays, true) @@ -102,6 +104,7 @@ FEATURE(scudo, LangOpts.Sanitize.hasOneOf(SanitizerKind::Scudo)) FEATURE(swiftasynccc, PP.getTargetInfo().checkCallingConvention(CC_SwiftAsync) == clang::TargetInfo::CCCR_OK) +FEATURE(pragma_stdc_cx_limited_range, true) // Objective-C features FEATURE(objc_arr, LangOpts.ObjCAutoRefCount) // FIXME: REMOVE? FEATURE(objc_arc, LangOpts.ObjCAutoRefCount) diff --git a/clang/include/clang/Basic/FileEntry.h b/clang/include/clang/Basic/FileEntry.h index 6351aeae92e2c457c1b196c095b27d9c291c37d2..35efa147950f0608476179f74d70742b69ca622b 100644 --- a/clang/include/clang/Basic/FileEntry.h +++ b/clang/include/clang/Basic/FileEntry.h @@ -279,72 +279,6 @@ template <> struct DenseMapInfo { namespace clang { -/// Wrapper around OptionalFileEntryRef that degrades to 'const FileEntry*', -/// facilitating incremental patches to propagate FileEntryRef. -/// -/// This class can be used as return value or field where it's convenient for -/// an OptionalFileEntryRef to degrade to a 'const FileEntry*'. The purpose -/// is to avoid code churn due to dances like the following: -/// \code -/// // Old code. -/// lvalue = rvalue; -/// -/// // Temporary code from an incremental patch. -/// OptionalFileEntryRef MaybeF = rvalue; -/// lvalue = MaybeF ? &MaybeF.getFileEntry() : nullptr; -/// -/// // Final code. -/// lvalue = rvalue; -/// \endcode -/// -/// FIXME: Once FileEntryRef is "everywhere" and FileEntry::LastRef and -/// FileEntry::getName have been deleted, delete this class and replace -/// instances with OptionalFileEntryRef. -class OptionalFileEntryRefDegradesToFileEntryPtr : public OptionalFileEntryRef { -public: - OptionalFileEntryRefDegradesToFileEntryPtr() = default; - OptionalFileEntryRefDegradesToFileEntryPtr( - OptionalFileEntryRefDegradesToFileEntryPtr &&) = default; - OptionalFileEntryRefDegradesToFileEntryPtr( - const OptionalFileEntryRefDegradesToFileEntryPtr &) = default; - OptionalFileEntryRefDegradesToFileEntryPtr & - operator=(OptionalFileEntryRefDegradesToFileEntryPtr &&) = default; - OptionalFileEntryRefDegradesToFileEntryPtr & - operator=(const OptionalFileEntryRefDegradesToFileEntryPtr &) = default; - - OptionalFileEntryRefDegradesToFileEntryPtr(std::nullopt_t) {} - OptionalFileEntryRefDegradesToFileEntryPtr(FileEntryRef Ref) - : OptionalFileEntryRef(Ref) {} - OptionalFileEntryRefDegradesToFileEntryPtr(OptionalFileEntryRef MaybeRef) - : OptionalFileEntryRef(MaybeRef) {} - - OptionalFileEntryRefDegradesToFileEntryPtr &operator=(std::nullopt_t) { - OptionalFileEntryRef::operator=(std::nullopt); - return *this; - } - OptionalFileEntryRefDegradesToFileEntryPtr &operator=(FileEntryRef Ref) { - OptionalFileEntryRef::operator=(Ref); - return *this; - } - OptionalFileEntryRefDegradesToFileEntryPtr & - operator=(OptionalFileEntryRef MaybeRef) { - OptionalFileEntryRef::operator=(MaybeRef); - return *this; - } - - /// Degrade to 'const FileEntry *' to allow FileEntry::LastRef and - /// FileEntry::getName have been deleted, delete this class and replace - /// instances with OptionalFileEntryRef - operator const FileEntry *() const { - return has_value() ? &(*this)->getFileEntry() : nullptr; - } -}; - -static_assert( - std::is_trivially_copyable< - OptionalFileEntryRefDegradesToFileEntryPtr>::value, - "OptionalFileEntryRefDegradesToFileEntryPtr should be trivially copyable"); - inline bool operator==(const FileEntry *LHS, const OptionalFileEntryRef &RHS) { return LHS == (RHS ? &RHS->getFileEntry() : nullptr); } diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def index c3d5399905a3fdae2580eadcfdf47c520513df14..152d9f65f86dbe022596239d664cc645a667ba79 100644 --- a/clang/include/clang/Basic/LangOptions.def +++ b/clang/include/clang/Basic/LangOptions.def @@ -220,6 +220,8 @@ BENIGN_LANGOPT(NoSignedZero , 1, 0, "Permit Floating Point optimization wit BENIGN_LANGOPT(AllowRecip , 1, 0, "Permit Floating Point reciprocal") BENIGN_LANGOPT(ApproxFunc , 1, 0, "Permit Floating Point approximation") +ENUM_LANGOPT(ComplexRange, ComplexRangeKind, 2, CX_Full, "Enable use of range reduction for complex arithmetics.") + BENIGN_LANGOPT(ObjCGCBitmapPrint , 1, 0, "printing of GC's bitmap layout for __weak/__strong ivars") BENIGN_LANGOPT(AccessControl , 1, 1, "C++ access control") diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h index 2d167dd2bdf1287a4a4fa8a8e7a84697b961ca29..9f986fce2d44188f0414034ab511a2de980e6ca2 100644 --- a/clang/include/clang/Basic/LangOptions.h +++ b/clang/include/clang/Basic/LangOptions.h @@ -152,6 +152,7 @@ public: MSVC2019 = 1920, MSVC2019_5 = 1925, MSVC2019_8 = 1928, + MSVC2022_3 = 1933, }; enum SYCLMajorVersion { @@ -391,6 +392,8 @@ public: IncompleteOnly = 3, }; + enum ComplexRangeKind { CX_Full, CX_Limited, CX_Fortran }; + public: /// The used language standard. LangStandard::Kind LangStd; @@ -740,6 +743,7 @@ public: setAllowFEnvAccess(true); else setAllowFEnvAccess(LangOptions::FPM_Off); + setComplexRange(LO.getComplexRange()); } bool allowFPContractWithinStatement() const { diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index d29cc0b45d583e049e614de254761d99ac2efb2d..62786e3ac865e640960279afeda93e333fab596a 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -156,7 +156,7 @@ public: /// The build directory of this module. This is the directory in /// which the module is notionally built, and relative to which its headers /// are found. - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr Directory; + OptionalDirectoryEntryRef Directory; /// The presumed file name for the module map defining this module. /// Only non-empty when building from preprocessed source. @@ -672,7 +672,7 @@ public: } /// The serialized AST file for this module, if one was created. - OptionalFileEntryRefDegradesToFileEntryPtr getASTFile() const { + OptionalFileEntryRef getASTFile() const { return getTopLevelModule()->ASTFile; } diff --git a/clang/include/clang/Basic/OpenACCKinds.h b/clang/include/clang/Basic/OpenACCKinds.h index 449a75638b43f5e8316043baf7b76cfe28c851fc..62c0a4c1a9dea41f09b14335cdb6b7022a8a82d7 100644 --- a/clang/include/clang/Basic/OpenACCKinds.h +++ b/clang/include/clang/Basic/OpenACCKinds.h @@ -53,7 +53,7 @@ enum class OpenACCDirectiveKind { Shutdown, Set, Update, - // FIXME: wait construct. + Wait, // Procedure Calls in Compute Regions. Routine, diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h index 985ea6354b82199f097fedb044c2829b5e9b1aea..d2ece14da0b11ab1e1448dd0ccf6515e8237994e 100644 --- a/clang/include/clang/Basic/SourceManager.h +++ b/clang/include/clang/Basic/SourceManager.h @@ -143,7 +143,7 @@ public: /// /// FIXME: Make non-optional using a virtual file as needed, remove \c /// Filename and use \c OrigEntry.getNameAsRequested() instead. - OptionalFileEntryRefDegradesToFileEntryPtr OrigEntry; + OptionalFileEntryRef OrigEntry; /// References the file which the contents were actually loaded from. /// @@ -1064,8 +1064,8 @@ public: /// Returns the FileEntry record for the provided FileID. const FileEntry *getFileEntryForID(FileID FID) const { - if (auto *Entry = getSLocEntryForFile(FID)) - return Entry->getFile().getContentCache().OrigEntry; + if (auto FE = getFileEntryRefForID(FID)) + return *FE; return nullptr; } @@ -1083,9 +1083,11 @@ public: std::optional getNonBuiltinFilenameForID(FileID FID) const; /// Returns the FileEntry record for the provided SLocEntry. - const FileEntry *getFileEntryForSLocEntry(const SrcMgr::SLocEntry &sloc) const - { - return sloc.getFile().getContentCache().OrigEntry; + const FileEntry * + getFileEntryForSLocEntry(const SrcMgr::SLocEntry &SLocEntry) const { + if (auto FE = SLocEntry.getFile().getContentCache().OrigEntry) + return *FE; + return nullptr; } /// Return a StringRef to the source buffer data for the diff --git a/clang/include/clang/Basic/SyncScope.h b/clang/include/clang/Basic/SyncScope.h index 7919f64c6daf97b0ac38a02247478f92bb10d3ca..bc7ec7b5cf777ef4556678445dc44fed078dcfb5 100644 --- a/clang/include/clang/Basic/SyncScope.h +++ b/clang/include/clang/Basic/SyncScope.h @@ -40,6 +40,11 @@ namespace clang { /// Update getAsString. /// enum class SyncScope { + SystemScope, + DeviceScope, + WorkgroupScope, + WavefrontScope, + SingleScope, HIPSingleThread, HIPWavefront, HIPWorkgroup, @@ -54,6 +59,16 @@ enum class SyncScope { inline llvm::StringRef getAsString(SyncScope S) { switch (S) { + case SyncScope::SystemScope: + return "system_scope"; + case SyncScope::DeviceScope: + return "device_scope"; + case SyncScope::WorkgroupScope: + return "workgroup_scope"; + case SyncScope::WavefrontScope: + return "wavefront_scope"; + case SyncScope::SingleScope: + return "single_scope"; case SyncScope::HIPSingleThread: return "hip_singlethread"; case SyncScope::HIPWavefront: @@ -77,7 +92,7 @@ inline llvm::StringRef getAsString(SyncScope S) { } /// Defines the kind of atomic scope models. -enum class AtomicScopeModelKind { None, OpenCL, HIP }; +enum class AtomicScopeModelKind { None, OpenCL, HIP, Generic }; /// Defines the interface for synch scope model. class AtomicScopeModel { @@ -205,6 +220,56 @@ public: } }; +/// Defines the generic atomic scope model. +class AtomicScopeGenericModel : public AtomicScopeModel { +public: + /// The enum values match predefined built-in macros __ATOMIC_SCOPE_*. + enum ID { + System = 0, + Device = 1, + Workgroup = 2, + Wavefront = 3, + Single = 4, + Last = Single + }; + + AtomicScopeGenericModel() = default; + + SyncScope map(unsigned S) const override { + switch (static_cast(S)) { + case Device: + return SyncScope::DeviceScope; + case System: + return SyncScope::SystemScope; + case Workgroup: + return SyncScope::WorkgroupScope; + case Wavefront: + return SyncScope::WavefrontScope; + case Single: + return SyncScope::SingleScope; + } + llvm_unreachable("Invalid language sync scope value"); + } + + bool isValid(unsigned S) const override { + return S >= static_cast(System) && + S <= static_cast(Last); + } + + ArrayRef getRuntimeValues() const override { + static_assert(Last == Single, "Does not include all sync scopes"); + static const unsigned Scopes[] = { + static_cast(Device), static_cast(System), + static_cast(Workgroup), static_cast(Wavefront), + static_cast(Single)}; + return llvm::ArrayRef(Scopes); + } + + unsigned getFallBackValue() const override { + return static_cast(System); + } +}; + inline std::unique_ptr AtomicScopeModel::create(AtomicScopeModelKind K) { switch (K) { @@ -214,6 +279,8 @@ AtomicScopeModel::create(AtomicScopeModelKind K) { return std::make_unique(); case AtomicScopeModelKind::HIP: return std::make_unique(); + case AtomicScopeModelKind::Generic: + return std::make_unique(); } llvm_unreachable("Invalid atomic scope model kind"); } diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h index 41f3c2e403cbef6d0fbd7c9640f810f418d7da43..ec0189627dfbd2c474803b19ae1ff5c45b6db9a6 100644 --- a/clang/include/clang/Basic/TargetInfo.h +++ b/clang/include/clang/Basic/TargetInfo.h @@ -266,7 +266,6 @@ protected: LLVM_PREFERRED_TYPE(bool) unsigned AllowAMDGPUUnsafeFPAtomics : 1; - LLVM_PREFERRED_TYPE(bool) unsigned ARMCDECoprocMask : 8; unsigned MaxOpenCLWorkGroupSize; diff --git a/clang/include/clang/Basic/TargetOSMacros.def b/clang/include/clang/Basic/TargetOSMacros.def new file mode 100644 index 0000000000000000000000000000000000000000..dfc2e033f6fd0db926d6882c92a25fc8bcfde07e --- /dev/null +++ b/clang/include/clang/Basic/TargetOSMacros.def @@ -0,0 +1,55 @@ +//===--- TargetOSMacros.def - Target OS macros ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file specifies the predefined TARGET_OS_* conditional macros. +// A target macro `Name` should be defined if `Predicate` evaluates to true. +// The macro expects `const llvm::Triple &Triple` and the class `llvm::Triple` +// to be available for the predicate. +// +//===----------------------------------------------------------------------===// + +#ifndef TARGET_OS +#define TARGET_OS(Name, Predicate) +#endif + +// Windows targets. +TARGET_OS(TARGET_OS_WIN32, Triple.isOSWindows()) +TARGET_OS(TARGET_OS_WINDOWS, Triple.isOSWindows()) + +// Linux target. +TARGET_OS(TARGET_OS_LINUX, Triple.isOSLinux()) + +// Unix target. +TARGET_OS(TARGET_OS_UNIX, Triple.isOSNetBSD() || + Triple.isOSFreeBSD() || + Triple.isOSOpenBSD() || + Triple.isOSSolaris()) + +// Apple (Mac) targets. +TARGET_OS(TARGET_OS_MAC, Triple.isOSDarwin()) +TARGET_OS(TARGET_OS_OSX, Triple.isMacOSX()) +TARGET_OS(TARGET_OS_IPHONE, Triple.isiOS() || Triple.isTvOS() || + Triple.isWatchOS()) +// Triple::isiOS() also includes tvOS +TARGET_OS(TARGET_OS_IOS, Triple.getOS() == llvm::Triple::IOS) +TARGET_OS(TARGET_OS_TV, Triple.isTvOS()) +TARGET_OS(TARGET_OS_WATCH, Triple.isWatchOS()) +TARGET_OS(TARGET_OS_DRIVERKIT, Triple.isDriverKit()) +TARGET_OS(TARGET_OS_MACCATALYST, Triple.isMacCatalystEnvironment()) +TARGET_OS(TARGET_OS_SIMULATOR, Triple.isSimulatorEnvironment()) + +// Deprecated Apple target conditionals. +TARGET_OS(TARGET_OS_EMBEDDED, (Triple.isiOS() || Triple.isTvOS() \ + || Triple.isWatchOS()) \ + && !Triple.isMacCatalystEnvironment() \ + && !Triple.isSimulatorEnvironment()) +TARGET_OS(TARGET_OS_NANO, Triple.isWatchOS()) +TARGET_OS(TARGET_IPHONE_SIMULATOR, Triple.isSimulatorEnvironment()) +TARGET_OS(TARGET_OS_UIKITFORMAC, Triple.isMacCatalystEnvironment()) + +#undef TARGET_OS diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def index 5f9915d210221c4f8619f581c030368908077a6b..3f0e1e1a7d45ad2cdcf1aba4c578736913fdc093 100644 --- a/clang/include/clang/Basic/TokenKinds.def +++ b/clang/include/clang/Basic/TokenKinds.def @@ -911,6 +911,11 @@ PRAGMA_ANNOTATION(pragma_fenv_access_ms) // handles them. PRAGMA_ANNOTATION(pragma_fenv_round) +// Annotation for #pragma STDC CX_LIMITED_RANGE +// The lexer produces these so that they only take effect when the parser +// handles them. +PRAGMA_ANNOTATION(pragma_cx_limited_range) + // Annotation for #pragma float_control // The lexer produces these so that they only take effect when the parser // handles them. diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td index 896c6f6e8033cdc7e15b2494bf3034f96c363f92..42ad7da737b4904f24a453bc4014cc3de1593c3c 100644 --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -1296,6 +1296,11 @@ def SVCREATE_3_BF16 : SInst<"svcreate3[_{d}]", "3ddd", "b", MergeNone, "", [IsT def SVCREATE_4_BF16 : SInst<"svcreate4[_{d}]", "4dddd", "b", MergeNone, "", [IsTupleCreate]>; } +let TargetGuard = "sve2p1" in { + def SVCREATE_2_B : SInst<"svcreate2[_{d}]", "2dd", "Pc", MergeNone, "", [IsTupleCreate]>; + def SVCREATE_4_B : SInst<"svcreate4[_{d}]", "4dddd", "Pc", MergeNone, "", [IsTupleCreate]>; +} + //////////////////////////////////////////////////////////////////////////////// // Vector insertion and extraction def SVGET_2 : SInst<"svget2[_{d}]", "d2i", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_1>]>; @@ -1316,6 +1321,13 @@ def SVSET_3_BF16 : SInst<"svset3[_{d}]", "33id", "b", MergeNone, "", [IsTupleSet def SVSET_4_BF16 : SInst<"svset4[_{d}]", "44id", "b", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_3>]>; } +let TargetGuard = "sve2p1" in { + def SVGET_2_B : SInst<"svget2[_{d}]", "d2i", "Pc", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_1>]>; + def SVGET_4_B : SInst<"svget4[_{d}]", "d4i", "Pc", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_3>]>; + + def SVSET_2_B : SInst<"svset2[_{d}]", "22id", "Pc", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_1>]>; + def SVSET_4_B : SInst<"svset4[_{d}]", "44id", "Pc", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_3>]>; +} //////////////////////////////////////////////////////////////////////////////// // SVE2 WhileGE/GT let TargetGuard = "sve2" in { @@ -1923,16 +1935,25 @@ def SVBGRP : SInst<"svbgrp[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sv def SVBGRP_N : SInst<"svbgrp[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bgrp_x">; } -let TargetGuard = "sve2p1" in { -def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [], []>; +let TargetGuard = "sve2p1|sme" in { +def SVPSEL_B : SInst<"svpsel_lane_b8", "PPPm", "Pc", MergeNone, "", [IsStreamingCompatible], []>; +def SVPSEL_H : SInst<"svpsel_lane_b16", "PPPm", "Ps", MergeNone, "", [IsStreamingCompatible], []>; +def SVPSEL_S : SInst<"svpsel_lane_b32", "PPPm", "Pi", MergeNone, "", [IsStreamingCompatible], []>; +def SVPSEL_D : SInst<"svpsel_lane_b64", "PPPm", "Pl", MergeNone, "", [IsStreamingCompatible], []>; +def SVPSEL_COUNT_ALIAS_B : SInst<"svpsel_lane_c8", "}}Pm", "Pc", MergeNone, "", [IsStreamingCompatible], []>; +def SVPSEL_COUNT_ALIAS_H : SInst<"svpsel_lane_c16", "}}Pm", "Ps", MergeNone, "", [IsStreamingCompatible], []>; +def SVPSEL_COUNT_ALIAS_S : SInst<"svpsel_lane_c32", "}}Pm", "Pi", MergeNone, "", [IsStreamingCompatible], []>; +def SVPSEL_COUNT_ALIAS_D : SInst<"svpsel_lane_c64", "}}Pm", "Pl", MergeNone, "", [IsStreamingCompatible], []>; +} -def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [], [ImmCheck<1, ImmCheck0_3>]>; -def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [], [ImmCheck<1, ImmCheck0_1>]>; +let TargetGuard = "sve2p1|sme2" in { +//FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available +def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>; +def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>; +} -def SVPSEL_COUNT_ALIAS_B : SInst<"svpsel_lane_c8", "}}Pm", "Pc", MergeNone, "", [], []>; -def SVPSEL_COUNT_ALIAS_H : SInst<"svpsel_lane_c16", "}}Pm", "Ps", MergeNone, "", [], []>; -def SVPSEL_COUNT_ALIAS_S : SInst<"svpsel_lane_c32", "}}Pm", "Pi", MergeNone, "", [], []>; -def SVPSEL_COUNT_ALIAS_D : SInst<"svpsel_lane_c64", "}}Pm", "Pl", MergeNone, "", [], []>; +let TargetGuard = "sve2p1" in { +def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [], []>; def SVWHILEGE_COUNT : SInst<"svwhilege_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; @@ -2033,11 +2054,6 @@ let TargetGuard = "sve2p1" in { def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sclamp", [], []>; def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [], []>; -def SVPSEL_B : SInst<"svpsel_lane_b8", "PPPm", "Pc", MergeNone, "", [], []>; -def SVPSEL_H : SInst<"svpsel_lane_b16", "PPPm", "Ps", MergeNone, "", [], []>; -def SVPSEL_S : SInst<"svpsel_lane_b32", "PPPm", "Pi", MergeNone, "", [], []>; -def SVPSEL_D : SInst<"svpsel_lane_b64", "PPPm", "Pl", MergeNone, "", [], []>; - def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone], [ImmCheck<1, ImmCheck2_4_Mul2>]>; defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUl", "aarch64_sve_revd">; @@ -2134,6 +2150,24 @@ let TargetGuard = "sme2" in { def SVURSHL_X2 : SInst<"svrshl[_{d}_x2]", "222", "UcUsUiUl", MergeNone, "aarch64_sve_urshl_x2", [IsStreaming], []>; def SVSRSHL_X4 : SInst<"svrshl[_{d}_x4]", "444", "csil", MergeNone, "aarch64_sve_srshl_x4", [IsStreaming], []>; def SVURSHL_X4 : SInst<"svrshl[_{d}_x4]", "444", "UcUsUiUl", MergeNone, "aarch64_sve_urshl_x4", [IsStreaming], []>; + + def SVQRSHRN_X4 : SInst<"svqrshrn[_n]_{0}[_{d}_x4]", "q4i", "il", MergeNone, "aarch64_sve_sqrshrn_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; + def SVUQRSHRN_X4 : SInst<"svqrshrn[_n]_{0}[_{d}_x4]", "b4i", "UiUl", MergeNone, "aarch64_sve_uqrshrn_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; + + // SQRSHR / UQRSHR + def SVQRSHR_X2 : SInst<"svqrshr[_n]_{0}[_{d}_x2]", "h2i", "i", MergeNone, "aarch64_sve_sqrshr_x2", [IsStreaming], [ImmCheck<1, ImmCheck1_16>]>; + def SVUQRSHR_X2 : SInst<"svqrshr[_n]_{0}[_{d}_x2]", "e2i", "Ui", MergeNone, "aarch64_sve_uqrshr_x2", [IsStreaming], [ImmCheck<1, ImmCheck1_16>]>; + def SVQRSHR_X4 : SInst<"svqrshr[_n]_{0}[_{d}_x4]", "q4i", "il", MergeNone, "aarch64_sve_sqrshr_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; + def SVUQRSHR_X4 : SInst<"svqrshr[_n]_{0}[_{d}_x4]", "b4i", "UiUl", MergeNone, "aarch64_sve_uqrshr_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; + + // SQRSHRU + def SVSQRSHRU_X2 : SInst<"svqrshru[_n]_{0}[_{d}_x2]", "e2i", "i", MergeNone, "aarch64_sve_sqrshru_x2", [IsStreaming], [ImmCheck<1, ImmCheck1_16>]>; + def SVSQRSHRU_X4 : SInst<"svqrshru[_n]_{0}[_{d}_x4]", "b4i", "il", MergeNone, "aarch64_sve_sqrshru_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; + + def SVSQRSHRUN_X4 : SInst<"svqrshrun[_n]_{0}[_{d}_x4]", "b4i", "il", MergeNone, "aarch64_sve_sqrshrun_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; + + def REINTERPRET_SVBOOL_TO_SVCOUNT : Inst<"svreinterpret[_c]", "}P", "Pc", MergeNone, "", [IsStreamingCompatible], []>; + def REINTERPRET_SVCOUNT_TO_SVBOOL : Inst<"svreinterpret[_b]", "P}", "Pc", MergeNone, "", [IsStreamingCompatible], []>; } let TargetGuard = "sve2p1" in { diff --git a/clang/include/clang/Driver/Multilib.h b/clang/include/clang/Driver/Multilib.h index 6a9533e6dd831f15ffd5acf3d5db6d4a50692adb..9a2cc9bb1ba134a1f1a47f690df1b6554bdc3f8d 100644 --- a/clang/include/clang/Driver/Multilib.h +++ b/clang/include/clang/Driver/Multilib.h @@ -130,7 +130,7 @@ public: /// Select compatible variants, \returns false if none are compatible bool select(const Multilib::flags_list &Flags, - llvm::SmallVector &) const; + llvm::SmallVectorImpl &) const; unsigned size() const { return Multilibs.size(); } diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 0eec2b352637623f1841216f5f72887e61a96220..25c76cf2ad2c84ad15c2ac6a39390043477afd91 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -1010,6 +1010,30 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block", NegFlag, BothFlags<[], [ClangOption], " that kernels are launched with uniform block sizes (default true for CUDA/HIP and false otherwise)">>; +def fcx_limited_range : Joined<["-"], "fcx-limited-range">, + Group, Visibility<[ClangOption, CC1Option]>, + HelpText<"Basic algebraic expansions of complex arithmetic operations " + "involving are enabled.">; + +def fno_cx_limited_range : Joined<["-"], "fno-cx-limited-range">, + Group, Visibility<[ClangOption, CC1Option]>, + HelpText<"Basic algebraic expansions of complex arithmetic operations " + "involving are disabled.">; + +def fcx_fortran_rules : Joined<["-"], "fcx-fortran-rules">, + Group, Visibility<[ClangOption, CC1Option]>, + HelpText<"Range reduction is enabled for complex arithmetic operations.">; + +def fno_cx_fortran_rules : Joined<["-"], "fno-cx-fortran-rules">, + Group, Visibility<[ClangOption, CC1Option]>, + HelpText<"Range reduction is disabled for complex arithmetic operations.">; + +def complex_range_EQ : Joined<["-"], "complex-range=">, Group, + Visibility<[CC1Option]>, + Values<"full,limited,fortran">, NormalizedValuesScope<"LangOptions">, + NormalizedValues<["CX_Full", "CX_Limited", "CX_Fortran"]>, + MarshallingInfoEnum, "CX_Full">; + // OpenCL-only Options def cl_opt_disable : Flag<["-"], "cl-opt-disable">, Group, Visibility<[ClangOption, CC1Option]>, @@ -1818,6 +1842,9 @@ def fcomment_block_commands : CommaJoined<["-"], "fcomment-block-commands=">, Gr Visibility<[ClangOption, CC1Option]>, HelpText<"Treat each comma separated argument in as a documentation comment block command">, MetaVarName<"">, MarshallingInfoStringVector>; +defm define_target_os_macros : OptInCC1FFlag<"define-target-os-macros", + "Enable", "Disable", " predefined target OS macros", + [ClangOption, CC1Option]>; def fparse_all_comments : Flag<["-"], "fparse-all-comments">, Group, Visibility<[ClangOption, CC1Option]>, MarshallingInfoFlag>; @@ -6351,6 +6378,12 @@ def J : JoinedOrSeparate<["-"], "J">, Group, Alias; +let Visibility = [FlangOption] in { +def no_fortran_main : Flag<["-"], "fno-fortran-main">, + Visibility<[FlangOption]>, Group, + HelpText<"Do not include Fortran_main.a (provided by Flang) when linking">; +} // let Visibility = [ FlangOption ] + //===----------------------------------------------------------------------===// // FC1 Options //===----------------------------------------------------------------------===// diff --git a/clang/include/clang/Frontend/FrontendActions.h b/clang/include/clang/Frontend/FrontendActions.h index 3940e00eeb8dba7924a04ae6e20a2a05daa29534..fcce31ac0590ff678adec7999a7cb5d0c3552cfd 100644 --- a/clang/include/clang/Frontend/FrontendActions.h +++ b/clang/include/clang/Frontend/FrontendActions.h @@ -151,6 +151,9 @@ class GenerateModuleInterfaceAction : public GenerateModuleAction { private: bool BeginSourceFileAction(CompilerInstance &CI) override; + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) override; + std::unique_ptr CreateOutputFile(CompilerInstance &CI, StringRef InFile) override; }; diff --git a/clang/include/clang/Lex/HeaderSearch.h b/clang/include/clang/Lex/HeaderSearch.h index a100598c80155fa5d5259bcd3601b0890c032744..a2c33842924b101b81b724decde96438be1bdd36 100644 --- a/clang/include/clang/Lex/HeaderSearch.h +++ b/clang/include/clang/Lex/HeaderSearch.h @@ -247,16 +247,13 @@ class HeaderSearch { /// \#include search path information. Requests for \#include "x" search the /// directory of the \#including file first, then each directory in SearchDirs /// consecutively. Requests for search the current dir first, then each - /// directory in SearchDirs, starting at AngledDirIdx, consecutively. If - /// NoCurDirSearch is true, then the check for the file in the current - /// directory is suppressed. + /// directory in SearchDirs, starting at AngledDirIdx, consecutively. std::vector SearchDirs; /// Whether the DirectoryLookup at the corresponding index in SearchDirs has /// been successfully used to lookup a file. std::vector SearchDirsUsage; unsigned AngledDirIdx = 0; unsigned SystemDirIdx = 0; - bool NoCurDirSearch = false; /// Maps HeaderMap keys to SearchDir indices. When HeaderMaps are used /// heavily, SearchDirs can start with thousands of HeaderMaps, so this Index @@ -373,7 +370,7 @@ public: /// Interface for setting the file search paths. void SetSearchPaths(std::vector dirs, unsigned angledDirIdx, - unsigned systemDirIdx, bool noCurDirSearch, + unsigned systemDirIdx, llvm::DenseMap searchDirToHSEntry); /// Add an additional search path. diff --git a/clang/include/clang/Lex/ModuleMap.h b/clang/include/clang/Lex/ModuleMap.h index 32e7e8f899e502cf6953f094121b16e91dc71177..867cb6eab42f2d7bf04bfa8e32c5e073170f7816 100644 --- a/clang/include/clang/Lex/ModuleMap.h +++ b/clang/include/clang/Lex/ModuleMap.h @@ -82,7 +82,7 @@ class ModuleMap { /// The directory used for Clang-supplied, builtin include headers, /// such as "stdint.h". - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr BuiltinIncludeDir; + OptionalDirectoryEntryRef BuiltinIncludeDir; /// Language options used to parse the module map itself. /// @@ -408,16 +408,12 @@ public: /// Set the target information. void setTarget(const TargetInfo &Target); - /// Set the directory that contains Clang-supplied include - /// files, such as our stdarg.h or tgmath.h. - void setBuiltinIncludeDir(DirectoryEntryRef Dir) { - BuiltinIncludeDir = Dir; - } + /// Set the directory that contains Clang-supplied include files, such as our + /// stdarg.h or tgmath.h. + void setBuiltinIncludeDir(DirectoryEntryRef Dir) { BuiltinIncludeDir = Dir; } /// Get the directory that contains Clang-supplied include files. - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr getBuiltinDir() const { - return BuiltinIncludeDir; - } + OptionalDirectoryEntryRef getBuiltinDir() const { return BuiltinIncludeDir; } /// Is this a compiler builtin header? bool isBuiltinHeader(FileEntryRef File); diff --git a/clang/include/clang/Lex/PreprocessorLexer.h b/clang/include/clang/Lex/PreprocessorLexer.h index eebaad7d50db3b1759c7ccb43b3b00d25e74e106..d71fe708ab20a20a7bd7928e474e835d5031076b 100644 --- a/clang/include/clang/Lex/PreprocessorLexer.h +++ b/clang/include/clang/Lex/PreprocessorLexer.h @@ -157,7 +157,7 @@ public: /// getFileEntry - Return the FileEntry corresponding to this FileID. Like /// getFileID(), this only works for lexers with attached preprocessors. - OptionalFileEntryRefDegradesToFileEntryPtr getFileEntry() const; + OptionalFileEntryRef getFileEntry() const; /// Iterator that traverses the current stack of preprocessor /// conditional directives (\#if/\#ifdef/\#ifndef). diff --git a/clang/include/clang/Lex/PreprocessorOptions.h b/clang/include/clang/Lex/PreprocessorOptions.h index 058194bcde72e51a8003b04ec4a48ac5df5e8df0..f841e4a028df50d3030a21145fa8d9af8bb047a3 100644 --- a/clang/include/clang/Lex/PreprocessorOptions.h +++ b/clang/include/clang/Lex/PreprocessorOptions.h @@ -76,6 +76,9 @@ public: /// predefines. bool UsePredefines = true; + /// Indicates whether to predefine target OS macros. + bool DefineTargetOSMacros = false; + /// Whether we should maintain a detailed record of all macro /// definitions and expansions. bool DetailedRecord = false; diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h index 32d0b76c35b0d7780db09dfc1d412e7473d56e54..2dbe090bd0932fc5afe59b25ffaf4f5910f2de70 100644 --- a/clang/include/clang/Parse/Parser.h +++ b/clang/include/clang/Parse/Parser.h @@ -769,6 +769,10 @@ private: /// #pragma STDC FENV_ROUND... void HandlePragmaFEnvRound(); + /// Handle the annotation token produced for + /// #pragma STDC CX_LIMITED_RANGE... + void HandlePragmaCXLimitedRange(); + /// Handle the annotation token produced for /// #pragma float_control void HandlePragmaFloatControl(); @@ -3544,6 +3548,7 @@ private: void ParseOpenACCCacheVarList(); /// Parses a single variable in a variable list for the 'cache' construct. bool ParseOpenACCCacheVar(); + bool ParseOpenACCWaitArgument(); private: //===--------------------------------------------------------------------===// diff --git a/clang/include/clang/Sema/HLSLExternalSemaSource.h b/clang/include/clang/Sema/HLSLExternalSemaSource.h index 4b6bc96f72e225c8e3d1790359294f3715c0e8f4..c0bfff327139f8c6a3b80a009ca8ddd7159622d6 100644 --- a/clang/include/clang/Sema/HLSLExternalSemaSource.h +++ b/clang/include/clang/Sema/HLSLExternalSemaSource.h @@ -30,9 +30,9 @@ class HLSLExternalSemaSource : public ExternalSemaSource { void defineHLSLVectorAlias(); void defineTrivialHLSLTypes(); - void forwardDeclareHLSLTypes(); + void defineHLSLTypesWithForwardDeclarations(); - void completeBufferType(CXXRecordDecl *Record); + void onCompletion(CXXRecordDecl *Record, CompletionFunction Fn); public: ~HLSLExternalSemaSource() override; diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 26f4e38a46e83cfc3b5c730baa8b4055532765ac..1902d098f3c25e3d27e9a4970842192f66f3d468 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -8561,6 +8561,10 @@ public: TemplateParameterList *Params, TemplateArgumentLoc &Arg); + void NoteTemplateLocation(const NamedDecl &Decl, + std::optional ParamRange = {}); + void NoteTemplateParameterLocation(const NamedDecl &Decl); + ExprResult BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg, QualType ParamType, @@ -11019,6 +11023,11 @@ public: /// \#pragma STDC FENV_ACCESS void ActOnPragmaFEnvAccess(SourceLocation Loc, bool IsEnabled); + /// ActOnPragmaCXLimitedRange - Called on well formed + /// \#pragma STDC CX_LIMITED_RANGE + void ActOnPragmaCXLimitedRange(SourceLocation Loc, + LangOptions::ComplexRangeKind Range); + /// Called on well formed '\#pragma clang fp' that has option 'exceptions'. void ActOnPragmaFPExceptions(SourceLocation Loc, LangOptions::FPExceptionModeKind); diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index 7eefdca6815cdadb81a8fd34bbdae8c2d0760800..9bb89ec9410911afb362830ffdd35ff1059af899 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -2415,12 +2415,7 @@ public: BitsUnpacker(BitsUnpacker &&) = delete; BitsUnpacker operator=(const BitsUnpacker &) = delete; BitsUnpacker operator=(BitsUnpacker &&) = delete; - ~BitsUnpacker() { -#ifndef NDEBUG - while (isValid()) - assert(!getNextBit() && "There are unprocessed bits!"); -#endif - } + ~BitsUnpacker() = default; void updateValue(uint32_t V) { Value = V; diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h index 48be8676cc26a4cc4a1e47e9a843dd6c655cf72a..9a14129d72ff33a7cafaf1082c33a9869af8e8f9 100644 --- a/clang/include/clang/Serialization/ModuleFile.h +++ b/clang/include/clang/Serialization/ModuleFile.h @@ -104,7 +104,7 @@ public: return File; } - OptionalFileEntryRefDegradesToFileEntryPtr getFile() const { + OptionalFileEntryRef getFile() const { if (auto *P = Val.getPointer()) return FileEntryRef(*P); return std::nullopt; @@ -123,8 +123,8 @@ public: /// other modules. class ModuleFile { public: - ModuleFile(ModuleKind Kind, unsigned Generation) - : Kind(Kind), Generation(Generation) {} + ModuleFile(ModuleKind Kind, FileEntryRef File, unsigned Generation) + : Kind(Kind), File(File), Generation(Generation) {} ~ModuleFile(); // === General information === @@ -176,7 +176,7 @@ public: bool DidReadTopLevelSubmodule = false; /// The file entry for the module file. - OptionalFileEntryRefDegradesToFileEntryPtr File; + FileEntryRef File; /// The signature of the module file, which may be used instead of the size /// and modification time to identify this particular file. diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp index c5c2edf1bfe3aba2337aa4a5afa57bf54f2cb476..527ea6042daa0344237ed90e46e8729520db2c3b 100644 --- a/clang/lib/AST/Decl.cpp +++ b/clang/lib/AST/Decl.cpp @@ -4150,6 +4150,7 @@ FunctionDecl::setFunctionTemplateSpecialization(ASTContext &C, assert(TSK != TSK_Undeclared && "Must specify the type of function template specialization"); assert((TemplateOrSpecialization.isNull() || + getFriendObjectKind() != FOK_None || TSK == TSK_ExplicitSpecialization) && "Member specialization must be an explicit specialization"); FunctionTemplateSpecializationInfo *Info = diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp index 55c6b732b7081f40c1584d14fb05949873978712..b125fc676da8419e737761b7db2a711df54adb46 100644 --- a/clang/lib/AST/Expr.cpp +++ b/clang/lib/AST/Expr.cpp @@ -4887,6 +4887,7 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) { case AO__atomic_load_n: return 2; + case AO__scoped_atomic_load_n: case AO__opencl_atomic_load: case AO__hip_atomic_load: case AO__c11_atomic_store: @@ -4921,6 +4922,26 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) { case AO__atomic_fetch_max: return 3; + case AO__scoped_atomic_load: + case AO__scoped_atomic_store: + case AO__scoped_atomic_store_n: + case AO__scoped_atomic_fetch_add: + case AO__scoped_atomic_fetch_sub: + case AO__scoped_atomic_fetch_and: + case AO__scoped_atomic_fetch_or: + case AO__scoped_atomic_fetch_xor: + case AO__scoped_atomic_fetch_nand: + case AO__scoped_atomic_add_fetch: + case AO__scoped_atomic_sub_fetch: + case AO__scoped_atomic_and_fetch: + case AO__scoped_atomic_or_fetch: + case AO__scoped_atomic_xor_fetch: + case AO__scoped_atomic_nand_fetch: + case AO__scoped_atomic_min_fetch: + case AO__scoped_atomic_max_fetch: + case AO__scoped_atomic_fetch_min: + case AO__scoped_atomic_fetch_max: + case AO__scoped_atomic_exchange_n: case AO__hip_atomic_exchange: case AO__hip_atomic_fetch_add: case AO__hip_atomic_fetch_sub: @@ -4942,6 +4963,7 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) { case AO__atomic_exchange: return 4; + case AO__scoped_atomic_exchange: case AO__c11_atomic_compare_exchange_strong: case AO__c11_atomic_compare_exchange_weak: return 5; @@ -4952,6 +4974,10 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) { case AO__atomic_compare_exchange: case AO__atomic_compare_exchange_n: return 6; + + case AO__scoped_atomic_compare_exchange: + case AO__scoped_atomic_compare_exchange_n: + return 7; } llvm_unreachable("unknown atomic op"); } diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index 986302e1fd225f2194880f02c49a5c5b2715443f..f035c1419f4c98a6e55992ed12435fac5f568fb0 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -641,6 +641,10 @@ namespace { return false; } + /// Whether we're in a context where [[msvc::constexpr]] evaluation is + /// permitted. See MSConstexprDocs for description of permitted contexts. + bool CanEvalMSConstexpr = false; + private: APValue &createLocal(APValue::LValueBase Base, const void *Key, QualType T, ScopeKind Scope); @@ -674,6 +678,19 @@ namespace { private: llvm::TimeTraceScope TimeScope; }; + + /// RAII object used to change the current ability of + /// [[msvc::constexpr]] evaulation. + struct MSConstexprContextRAII { + CallStackFrame &Frame; + bool OldValue; + explicit MSConstexprContextRAII(CallStackFrame &Frame, bool Value) + : Frame(Frame), OldValue(Frame.CanEvalMSConstexpr) { + Frame.CanEvalMSConstexpr = Value; + } + + ~MSConstexprContextRAII() { Frame.CanEvalMSConstexpr = OldValue; } + }; } static bool HandleDestruction(EvalInfo &Info, const Expr *E, @@ -5546,11 +5563,14 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info, case Stmt::LabelStmtClass: return EvaluateStmt(Result, Info, cast(S)->getSubStmt(), Case); - case Stmt::AttributedStmtClass: - // As a general principle, C++11 attributes can be ignored without - // any semantic impact. - return EvaluateStmt(Result, Info, cast(S)->getSubStmt(), - Case); + case Stmt::AttributedStmtClass: { + const auto *AS = cast(S); + const auto *SS = AS->getSubStmt(); + MSConstexprContextRAII ConstexprContext( + *Info.CurrentCall, hasSpecificAttr(AS->getAttrs()) && + isa(SS)); + return EvaluateStmt(Result, Info, SS, Case); + } case Stmt::CaseStmtClass: case Stmt::DefaultStmtClass: @@ -5621,7 +5641,9 @@ static bool CheckConstexprFunction(EvalInfo &Info, SourceLocation CallLoc, } // Can we evaluate this function call? - if (Definition && Definition->isConstexpr() && Body) + if (Definition && Body && + (Definition->isConstexpr() || (Info.CurrentCall->CanEvalMSConstexpr && + Definition->hasAttr()))) return true; if (Info.getLangOpts().CPlusPlus11) { diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index f45e8624a7741a536c82564f910a698387a1f8ed..f7f8e6c73d84e21369343ff5eac81602a725480c 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -15,7 +15,6 @@ #include "Function.h" #include "PrimType.h" #include "Program.h" -#include "State.h" using namespace clang; using namespace clang::interp; diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h index 4f7778bdd2ff33371db54d2f9d7c5b444a302e35..6cf122f2ba55e1b82edd9ab2f53bde21525a2837 100644 --- a/clang/lib/AST/Interp/Interp.h +++ b/clang/lib/AST/Interp/Interp.h @@ -1619,7 +1619,11 @@ bool CastFloatingIntegral(InterpState &S, CodePtr OpPC) { QualType Type = E->getType(); S.CCEDiag(E, diag::note_constexpr_overflow) << F.getAPFloat() << Type; - return S.noteUndefinedBehavior(); + if (S.noteUndefinedBehavior()) { + S.Stk.push(T(Result)); + return true; + } + return false; } S.Stk.push(T(Result)); diff --git a/clang/lib/AST/Interp/InterpBuiltin.cpp b/clang/lib/AST/Interp/InterpBuiltin.cpp index 9cf206ecc212adba28c6f4a9b346901434171720..4384ace6b6be5e43d8a270c80164a42c2213ed2d 100644 --- a/clang/lib/AST/Interp/InterpBuiltin.cpp +++ b/clang/lib/AST/Interp/InterpBuiltin.cpp @@ -34,6 +34,19 @@ PrimType getIntPrimType(const InterpState &S) { llvm_unreachable("Int isn't 16 or 32 bit?"); } +PrimType getLongPrimType(const InterpState &S) { + const TargetInfo &TI = S.getCtx().getTargetInfo(); + unsigned LongWidth = TI.getLongWidth(); + + if (LongWidth == 64) + return PT_Sint64; + else if (LongWidth == 32) + return PT_Sint32; + else if (LongWidth == 16) + return PT_Sint16; + llvm_unreachable("long isn't 16, 32 or 64 bit?"); +} + /// Peek an integer value from the stack into an APSInt. static APSInt peekToAPSInt(InterpStack &Stk, PrimType T, size_t Offset = 0) { if (Offset == 0) @@ -110,6 +123,19 @@ static void pushAPSInt(InterpState &S, const APSInt &Val) { } } +/// Pushes \p Val to the stack, as a target-dependent 'long'. +static void pushLong(InterpState &S, int64_t Val) { + PrimType LongType = getLongPrimType(S); + if (LongType == PT_Sint64) + S.Stk.push>(Integral<64, true>::from(Val)); + else if (LongType == PT_Sint32) + S.Stk.push>(Integral<32, true>::from(Val)); + else if (LongType == PT_Sint16) + S.Stk.push>(Integral<16, true>::from(Val)); + else + llvm_unreachable("Long isn't 16, 32 or 64 bit?"); +} + static void pushSizeT(InterpState &S, uint64_t Val) { const TargetInfo &TI = S.getCtx().getTargetInfo(); unsigned SizeTWidth = TI.getTypeWidth(TI.getSizeType()); @@ -533,6 +559,26 @@ static bool interp__builtin_classify_type(InterpState &S, CodePtr OpPC, return true; } +// __builtin_expect(long, long) +// __builtin_expect_with_probability(long, long, double) +static bool interp__builtin_expect(InterpState &S, CodePtr OpPC, + const InterpFrame *Frame, + const Function *Func, const CallExpr *Call) { + // The return value is simply the value of the first parameter. + // We ignore the probability. + unsigned NumArgs = Call->getNumArgs(); + assert(NumArgs == 2 || NumArgs == 3); + + PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType()); + unsigned Offset = align(primSize(getLongPrimType(S))) * 2; + if (NumArgs == 3) + Offset += align(primSize(PT_Float)); + + APSInt Val = peekToAPSInt(S.Stk, ArgT, Offset); + pushLong(S, Val.getSExtValue()); + return true; +} + bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F, const CallExpr *Call) { InterpFrame *Frame = S.Current; @@ -702,6 +748,12 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F, return false; break; + case Builtin::BI__builtin_expect: + case Builtin::BI__builtin_expect_with_probability: + if (!interp__builtin_expect(S, OpPC, Frame, F, Call)) + return false; + break; + default: return false; } diff --git a/clang/lib/AST/Interp/InterpFrame.cpp b/clang/lib/AST/Interp/InterpFrame.cpp index b06923114c7a24ee5587f8e1f7ce18385ce70747..d460d7ea3710a88e0f99c19f10ecb0acce29fddc 100644 --- a/clang/lib/AST/Interp/InterpFrame.cpp +++ b/clang/lib/AST/Interp/InterpFrame.cpp @@ -228,7 +228,7 @@ Pointer InterpFrame::getParamPointer(unsigned Off) { SourceInfo InterpFrame::getSource(CodePtr PC) const { // Implicitly created functions don't have any code we could point at, // so return the call site. - if (Func && Func->getDecl()->isImplicit() && Caller) + if (Func && (!Func->hasBody() || Func->getDecl()->isImplicit()) && Caller) return Caller->getSource(RetPC); return S.getSource(Func, PC); @@ -243,7 +243,7 @@ SourceLocation InterpFrame::getLocation(CodePtr PC) const { } SourceRange InterpFrame::getRange(CodePtr PC) const { - if (Func && Func->getDecl()->isImplicit() && Caller) + if (Func && (!Func->hasBody() || Func->getDecl()->isImplicit()) && Caller) return Caller->getRange(RetPC); return S.getRange(Func, PC); diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp index 50ab6ea59be9d0311e7f5c84a372e93d2c8eb414..c59a66e103a6e3b929fa950f93414bd58c185de0 100644 --- a/clang/lib/AST/MicrosoftMangle.cpp +++ b/clang/lib/AST/MicrosoftMangle.cpp @@ -3809,14 +3809,14 @@ void MicrosoftMangleContextImpl::mangleCXXRTTICompleteObjectLocator( llvm::raw_svector_ostream Stream(VFTableMangling); mangleCXXVFTable(Derived, BasePath, Stream); - if (VFTableMangling.startswith("??@")) { - assert(VFTableMangling.endswith("@")); + if (VFTableMangling.starts_with("??@")) { + assert(VFTableMangling.ends_with("@")); Out << VFTableMangling << "??_R4@"; return; } - assert(VFTableMangling.startswith("??_7") || - VFTableMangling.startswith("??_S")); + assert(VFTableMangling.starts_with("??_7") || + VFTableMangling.starts_with("??_S")); Out << "??_R4" << VFTableMangling.str().drop_front(4); } diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp index a51c8b938f411c066655a5d478c72178388fb51b..706991f4fb501c493a07bc861fd9028832ea0d99 100644 --- a/clang/lib/AST/RecordLayoutBuilder.cpp +++ b/clang/lib/AST/RecordLayoutBuilder.cpp @@ -2942,8 +2942,8 @@ void MicrosoftRecordLayoutBuilder::layoutNonVirtualBase( } if (!FoundBase) { - if (MDCUsesEBO && BaseDecl->isEmpty()) { - assert(BaseLayout.getNonVirtualSize() == CharUnits::Zero()); + if (MDCUsesEBO && BaseDecl->isEmpty() && + (BaseLayout.getNonVirtualSize() == CharUnits::Zero())) { BaseOffset = CharUnits::Zero(); } else { // Otherwise, lay the base out at the end of the MDC. diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp index ab4a013de5f552c2f04e9a511de01bea3c021e9f..c04cb313c3387a3bd3169a61360e4b422e2eb5af 100644 --- a/clang/lib/AST/StmtPrinter.cpp +++ b/clang/lib/AST/StmtPrinter.cpp @@ -1841,6 +1841,7 @@ void StmtPrinter::VisitAtomicExpr(AtomicExpr *Node) { PrintExpr(Node->getPtr()); if (Node->getOp() != AtomicExpr::AO__c11_atomic_load && Node->getOp() != AtomicExpr::AO__atomic_load_n && + Node->getOp() != AtomicExpr::AO__scoped_atomic_load_n && Node->getOp() != AtomicExpr::AO__opencl_atomic_load && Node->getOp() != AtomicExpr::AO__hip_atomic_load) { OS << ", "; diff --git a/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp b/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp index 7430ef599e032d0c897461a777903d099948932f..2a7bfce5350159dfd6503f041b679a9533a3fb97 100644 --- a/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp +++ b/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp @@ -351,9 +351,10 @@ private: if (Invalid) return; - static constexpr unsigned Missing = -1; // TokenInfo stores the BB and set of elements that a token is part of. struct TokenInfo { + enum : unsigned { Missing = static_cast(-1) }; + // The basic block this is part of. // This is the BB of the stmt with the smallest containing range. unsigned BB = Missing; diff --git a/clang/lib/Analysis/ThreadSafety.cpp b/clang/lib/Analysis/ThreadSafety.cpp index 7fdf22c2f3919cb34134efb7a3decf9531578329..e25b843c9bf83e2c93d706e64a6a0741cc034330 100644 --- a/clang/lib/Analysis/ThreadSafety.cpp +++ b/clang/lib/Analysis/ThreadSafety.cpp @@ -1010,6 +1010,8 @@ class ThreadSafetyAnalyzer { ThreadSafetyHandler &Handler; const FunctionDecl *CurrentFunction; LocalVariableMap LocalVarMap; + // Maps constructed objects to `this` placeholder prior to initialization. + llvm::SmallDenseMap ConstructedObjects; FactManager FactMan; std::vector BlockInfo; @@ -1543,8 +1545,6 @@ class BuildLockset : public ConstStmtVisitor { FactSet FSet; // The fact set for the function on exit. const FactSet &FunctionExitFSet; - /// Maps constructed objects to `this` placeholder prior to initialization. - llvm::SmallDenseMap ConstructedObjects; LocalVariableMap::Context LVarCtx; unsigned CtxIndex; @@ -1808,7 +1808,7 @@ void BuildLockset::handleCall(const Expr *Exp, const NamedDecl *D, std::pair Placeholder = Analyzer->SxBuilder.createThisPlaceholder(Exp); [[maybe_unused]] auto inserted = - ConstructedObjects.insert({Exp, Placeholder.first}); + Analyzer->ConstructedObjects.insert({Exp, Placeholder.first}); assert(inserted.second && "Are we visiting the same expression again?"); if (isa(Exp)) Self = Placeholder.first; @@ -2128,10 +2128,10 @@ void BuildLockset::VisitDeclStmt(const DeclStmt *S) { E = EWC->getSubExpr()->IgnoreParens(); E = UnpackConstruction(E); - if (auto Object = ConstructedObjects.find(E); - Object != ConstructedObjects.end()) { + if (auto Object = Analyzer->ConstructedObjects.find(E); + Object != Analyzer->ConstructedObjects.end()) { Object->second->setClangDecl(VD); - ConstructedObjects.erase(Object); + Analyzer->ConstructedObjects.erase(Object); } } } @@ -2140,11 +2140,11 @@ void BuildLockset::VisitDeclStmt(const DeclStmt *S) { void BuildLockset::VisitMaterializeTemporaryExpr( const MaterializeTemporaryExpr *Exp) { if (const ValueDecl *ExtD = Exp->getExtendingDecl()) { - if (auto Object = - ConstructedObjects.find(UnpackConstruction(Exp->getSubExpr())); - Object != ConstructedObjects.end()) { + if (auto Object = Analyzer->ConstructedObjects.find( + UnpackConstruction(Exp->getSubExpr())); + Object != Analyzer->ConstructedObjects.end()) { Object->second->setClangDecl(ExtD); - ConstructedObjects.erase(Object); + Analyzer->ConstructedObjects.erase(Object); } } } @@ -2487,15 +2487,15 @@ void ThreadSafetyAnalyzer::runAnalysis(AnalysisDeclContext &AC) { // Clean up constructed object even if there are no attributes to // keep the number of objects in limbo as small as possible. - if (auto Object = LocksetBuilder.ConstructedObjects.find( + if (auto Object = ConstructedObjects.find( TD.getBindTemporaryExpr()->getSubExpr()); - Object != LocksetBuilder.ConstructedObjects.end()) { + Object != ConstructedObjects.end()) { const auto *DD = TD.getDestructorDecl(AC.getASTContext()); if (DD->hasAttrs()) // TODO: the location here isn't quite correct. LocksetBuilder.handleCall(nullptr, DD, Object->second, TD.getBindTemporaryExpr()->getEndLoc()); - LocksetBuilder.ConstructedObjects.erase(Object); + ConstructedObjects.erase(Object); } break; } diff --git a/clang/lib/Analysis/UninitializedValues.cpp b/clang/lib/Analysis/UninitializedValues.cpp index b796f7674cc1daeed867198289c594a87e1e8e32..e9111ded64eb1f8b46969bd5b1986f1f16e1839e 100644 --- a/clang/lib/Analysis/UninitializedValues.cpp +++ b/clang/lib/Analysis/UninitializedValues.cpp @@ -64,7 +64,7 @@ static bool isTrackedVar(const VarDecl *vd, const DeclContext *dc) { QualType ty = vd->getType(); if (const auto *RD = ty->getAsRecordDecl()) return recordIsNotEmpty(RD); - return ty->isScalarType() || ty->isVectorType() || ty->isRVVType(); + return ty->isScalarType() || ty->isVectorType() || ty->isRVVSizelessBuiltinType(); } return false; } diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp index e332a3609290aace02af6ea0e023dd21d18d0cb8..a1efb76be68b7d7fe05b5f7b76d9417481d69c0b 100644 --- a/clang/lib/Analysis/UnsafeBufferUsage.cpp +++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp @@ -1028,6 +1028,46 @@ public: } }; +// Representing a pointer type expression of the form `Ptr += n` in an +// Unspecified Untyped Context (UUC): +class UUCAddAssignGadget : public FixableGadget { +private: + static constexpr const char *const UUCAddAssignTag = + "PointerAddAssignUnderUUC"; + static constexpr const char *const OffsetTag = "Offset"; + + const BinaryOperator *Node; // the `Ptr += n` node + const Expr *Offset = nullptr; + +public: + UUCAddAssignGadget(const MatchFinder::MatchResult &Result) + : FixableGadget(Kind::UUCAddAssign), + Node(Result.Nodes.getNodeAs(UUCAddAssignTag)), + Offset(Result.Nodes.getNodeAs(OffsetTag)) { + assert(Node != nullptr && "Expecting a non-null matching result"); + } + + static bool classof(const Gadget *G) { + return G->getKind() == Kind::UUCAddAssign; + } + + static Matcher matcher() { + return stmt(isInUnspecifiedUntypedContext(expr(ignoringImpCasts( + binaryOperator(hasOperatorName("+="), + hasLHS(declRefExpr(toSupportedVariable())), + hasRHS(expr().bind(OffsetTag))) + .bind(UUCAddAssignTag))))); + } + + virtual std::optional getFixits(const Strategy &S) const override; + + virtual const Stmt *getBaseStmt() const override { return Node; } + + virtual DeclUseList getClaimedVarUseSites() const override { + return {dyn_cast(Node->getLHS())}; + } +}; + // Representing a fixable expression of the form `*(ptr + 123)` or `*(123 + // ptr)`: class DerefSimplePtrArithFixableGadget : public FixableGadget { @@ -1312,6 +1352,16 @@ PointerInitGadget::getFixits(const Strategy &S) const { return std::nullopt; } +static bool isNonNegativeIntegerExpr(const Expr *Expr, const VarDecl *VD, + const ASTContext &Ctx) { + if (auto ConstVal = Expr->getIntegerConstantExpr(Ctx)) { + if (ConstVal->isNegative()) + return false; + } else if (!Expr->getType()->isUnsignedIntegerType()) + return false; + return true; +} + std::optional ULCArraySubscriptGadget::getFixits(const Strategy &S) const { if (const auto *DRE = @@ -1319,14 +1369,12 @@ ULCArraySubscriptGadget::getFixits(const Strategy &S) const { if (const auto *VD = dyn_cast(DRE->getDecl())) { switch (S.lookup(VD)) { case Strategy::Kind::Span: { + // If the index has a negative constant value, we give up as no valid // fix-it can be generated: const ASTContext &Ctx = // FIXME: we need ASTContext to be passed in! VD->getASTContext(); - if (auto ConstVal = Node->getIdx()->getIntegerConstantExpr(Ctx)) { - if (ConstVal->isNegative()) - return std::nullopt; - } else if (!Node->getIdx()->getType()->isUnsignedIntegerType()) + if (!isNonNegativeIntegerExpr(Node->getIdx(), VD, Ctx)) return std::nullopt; // no-op is a good fix-it, otherwise return FixItList{}; @@ -1405,10 +1453,8 @@ static std::optional getPastLoc(const NodeTy *Node, const LangOptions &LangOpts) { SourceLocation Loc = Lexer::getLocForEndOfToken(Node->getEndLoc(), 0, SM, LangOpts); - if (Loc.isValid()) return Loc; - return std::nullopt; } @@ -1766,6 +1812,47 @@ fixUPCAddressofArraySubscriptWithSpan(const UnaryOperator *Node) { FixItHint::CreateReplacement(Node->getSourceRange(), SS.str())}; } +std::optional +UUCAddAssignGadget::getFixits(const Strategy &S) const { + DeclUseList DREs = getClaimedVarUseSites(); + + if (DREs.size() != 1) + return std::nullopt; // In cases of `Ptr += n` where `Ptr` is not a DRE, we + // give up + if (const VarDecl *VD = dyn_cast(DREs.front()->getDecl())) { + if (S.lookup(VD) == Strategy::Kind::Span) { + FixItList Fixes; + + const Stmt *AddAssignNode = getBaseStmt(); + StringRef varName = VD->getName(); + const ASTContext &Ctx = VD->getASTContext(); + + if (!isNonNegativeIntegerExpr(Offset, VD, Ctx)) + return std::nullopt; + + // To transform UUC(p += n) to UUC(p = p.subspan(..)): + bool NotParenExpr = + (Offset->IgnoreParens()->getBeginLoc() == Offset->getBeginLoc()); + std::string SS = varName.str() + " = " + varName.str() + ".subspan"; + if (NotParenExpr) + SS += "("; + + std::optional AddAssignLocation = getEndCharLoc( + AddAssignNode, Ctx.getSourceManager(), Ctx.getLangOpts()); + if (!AddAssignLocation) + return std::nullopt; + + Fixes.push_back(FixItHint::CreateReplacement( + SourceRange(AddAssignNode->getBeginLoc(), Node->getOperatorLoc()), + SS)); + if (NotParenExpr) + Fixes.push_back(FixItHint::CreateInsertion( + Offset->getEndLoc().getLocWithOffset(1), ")")); + return Fixes; + } + } + return std::nullopt; // Not in the cases that we can handle for now, give up. +} std::optional UPCPreIncrementGadget::getFixits(const Strategy &S) const { DeclUseList DREs = getClaimedVarUseSites(); diff --git a/clang/lib/Basic/Cuda.cpp b/clang/lib/Basic/Cuda.cpp index 65840b9f20252b62bee800895fa7dd991735b5cc..1b1da6a1356f2c4de266660eb57882388a8dd918 100644 --- a/clang/lib/Basic/Cuda.cpp +++ b/clang/lib/Basic/Cuda.cpp @@ -39,6 +39,8 @@ static const CudaVersionMapEntry CudaNameVersionMap[] = { CUDA_ENTRY(11, 8), CUDA_ENTRY(12, 0), CUDA_ENTRY(12, 1), + CUDA_ENTRY(12, 2), + CUDA_ENTRY(12, 3), {"", CudaVersion::NEW, llvm::VersionTuple(std::numeric_limits::max())}, {"unknown", CudaVersion::UNKNOWN, {}} // End of list tombstone. }; @@ -93,6 +95,7 @@ static const CudaArchToStringMap arch_names[] = { SM(87), // Jetson/Drive AGX Orin SM(89), // Ada Lovelace SM(90), // Hopper + SM(90a), // Hopper GFX(600), // gfx600 GFX(601), // gfx601 GFX(602), // gfx602 @@ -209,6 +212,8 @@ CudaVersion MinVersionForCudaArch(CudaArch A) { case CudaArch::SM_89: case CudaArch::SM_90: return CudaVersion::CUDA_118; + case CudaArch::SM_90a: + return CudaVersion::CUDA_120; default: llvm_unreachable("invalid enum"); } diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp index e4ac1abf12a7f8e340aabd49802ba83b34afa9f4..7523e509a47108c207559585d0e988371c1532ef 100644 --- a/clang/lib/Basic/Module.cpp +++ b/clang/lib/Basic/Module.cpp @@ -89,7 +89,7 @@ static bool isPlatformEnvironment(const TargetInfo &Target, StringRef Feature) { // where both are valid examples of the same platform+environment but in the // variant (2) the simulator is hardcoded as part of the platform name. Both // forms above should match for "iossimulator" requirement. - if (Target.getTriple().isOSDarwin() && PlatformEnv.endswith("simulator")) + if (Target.getTriple().isOSDarwin() && PlatformEnv.ends_with("simulator")) return PlatformEnv == Feature || CmpPlatformEnv(PlatformEnv, Feature); return PlatformEnv == Feature; diff --git a/clang/lib/Basic/Targets/AMDGPU.cpp b/clang/lib/Basic/Targets/AMDGPU.cpp index 409ae32ab4242151c22e223a94e2909d8ae5bd05..719fc51bfc286fb57247ea71b2c7626ce49e7e83 100644 --- a/clang/lib/Basic/Targets/AMDGPU.cpp +++ b/clang/lib/Basic/Targets/AMDGPU.cpp @@ -37,50 +37,50 @@ static const char *const DataLayoutStringAMDGCN = "-ni:7:8"; const LangASMap AMDGPUTargetInfo::AMDGPUDefIsGenMap = { - Generic, // Default - Global, // opencl_global - Local, // opencl_local - Constant, // opencl_constant - Private, // opencl_private - Generic, // opencl_generic - Global, // opencl_global_device - Global, // opencl_global_host - Global, // cuda_device - Constant, // cuda_constant - Local, // cuda_shared - Global, // sycl_global - Global, // sycl_global_device - Global, // sycl_global_host - Local, // sycl_local - Private, // sycl_private - Generic, // ptr32_sptr - Generic, // ptr32_uptr - Generic, // ptr64 - Generic, // hlsl_groupshared + llvm::AMDGPUAS::FLAT_ADDRESS, // Default + llvm::AMDGPUAS::GLOBAL_ADDRESS, // opencl_global + llvm::AMDGPUAS::LOCAL_ADDRESS, // opencl_local + llvm::AMDGPUAS::CONSTANT_ADDRESS, // opencl_constant + llvm::AMDGPUAS::PRIVATE_ADDRESS, // opencl_private + llvm::AMDGPUAS::FLAT_ADDRESS, // opencl_generic + llvm::AMDGPUAS::GLOBAL_ADDRESS, // opencl_global_device + llvm::AMDGPUAS::GLOBAL_ADDRESS, // opencl_global_host + llvm::AMDGPUAS::GLOBAL_ADDRESS, // cuda_device + llvm::AMDGPUAS::CONSTANT_ADDRESS, // cuda_constant + llvm::AMDGPUAS::LOCAL_ADDRESS, // cuda_shared + llvm::AMDGPUAS::GLOBAL_ADDRESS, // sycl_global + llvm::AMDGPUAS::GLOBAL_ADDRESS, // sycl_global_device + llvm::AMDGPUAS::GLOBAL_ADDRESS, // sycl_global_host + llvm::AMDGPUAS::LOCAL_ADDRESS, // sycl_local + llvm::AMDGPUAS::PRIVATE_ADDRESS, // sycl_private + llvm::AMDGPUAS::FLAT_ADDRESS, // ptr32_sptr + llvm::AMDGPUAS::FLAT_ADDRESS, // ptr32_uptr + llvm::AMDGPUAS::FLAT_ADDRESS, // ptr64 + llvm::AMDGPUAS::FLAT_ADDRESS, // hlsl_groupshared }; const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = { - Private, // Default - Global, // opencl_global - Local, // opencl_local - Constant, // opencl_constant - Private, // opencl_private - Generic, // opencl_generic - Global, // opencl_global_device - Global, // opencl_global_host - Global, // cuda_device - Constant, // cuda_constant - Local, // cuda_shared + llvm::AMDGPUAS::PRIVATE_ADDRESS, // Default + llvm::AMDGPUAS::GLOBAL_ADDRESS, // opencl_global + llvm::AMDGPUAS::LOCAL_ADDRESS, // opencl_local + llvm::AMDGPUAS::CONSTANT_ADDRESS, // opencl_constant + llvm::AMDGPUAS::PRIVATE_ADDRESS, // opencl_private + llvm::AMDGPUAS::FLAT_ADDRESS, // opencl_generic + llvm::AMDGPUAS::GLOBAL_ADDRESS, // opencl_global_device + llvm::AMDGPUAS::GLOBAL_ADDRESS, // opencl_global_host + llvm::AMDGPUAS::GLOBAL_ADDRESS, // cuda_device + llvm::AMDGPUAS::CONSTANT_ADDRESS, // cuda_constant + llvm::AMDGPUAS::LOCAL_ADDRESS, // cuda_shared // SYCL address space values for this map are dummy - Generic, // sycl_global - Generic, // sycl_global_device - Generic, // sycl_global_host - Generic, // sycl_local - Generic, // sycl_private - Generic, // ptr32_sptr - Generic, // ptr32_uptr - Generic, // ptr64 - Generic, // hlsl_groupshared + llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_global + llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_global_device + llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_global_host + llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_local + llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_private + llvm::AMDGPUAS::FLAT_ADDRESS, // ptr32_sptr + llvm::AMDGPUAS::FLAT_ADDRESS, // ptr32_uptr + llvm::AMDGPUAS::FLAT_ADDRESS, // ptr64 + llvm::AMDGPUAS::FLAT_ADDRESS, // hlsl_groupshared }; } // namespace targets diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h index 300d9691d8a0f2207ebc6eb64bd5626c12e55a17..1819ba544ccf84b51cff4c24bf9bed42fdcd327b 100644 --- a/clang/lib/Basic/Targets/AMDGPU.h +++ b/clang/lib/Basic/Targets/AMDGPU.h @@ -17,6 +17,7 @@ #include "clang/Basic/TargetInfo.h" #include "clang/Basic/TargetOptions.h" #include "llvm/ADT/StringSet.h" +#include "llvm/Support/AMDGPUAddrSpace.h" #include "llvm/Support/Compiler.h" #include "llvm/TargetParser/TargetParser.h" #include "llvm/TargetParser/Triple.h" @@ -29,13 +30,6 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo { static const char *const GCCRegNames[]; - enum AddrSpace { - Generic = 0, - Global = 1, - Local = 3, - Constant = 4, - Private = 5 - }; static const LangASMap AMDGPUDefIsGenMap; static const LangASMap AMDGPUDefIsPrivMap; @@ -106,7 +100,8 @@ public: return 32; unsigned TargetAS = getTargetAddressSpace(AS); - if (TargetAS == Private || TargetAS == Local) + if (TargetAS == llvm::AMDGPUAS::PRIVATE_ADDRESS || + TargetAS == llvm::AMDGPUAS::LOCAL_ADDRESS) return 32; return 64; @@ -376,7 +371,7 @@ public: } std::optional getConstantAddressSpace() const override { - return getLangASFromTargetAS(Constant); + return getLangASFromTargetAS(llvm::AMDGPUAS::CONSTANT_ADDRESS); } const llvm::omp::GV &getGridValue() const override { @@ -392,7 +387,7 @@ public: /// \returns Target specific vtbl ptr address space. unsigned getVtblPtrAddressSpace() const override { - return static_cast(Constant); + return static_cast(llvm::AMDGPUAS::CONSTANT_ADDRESS); } /// \returns If a target requires an address within a target specific address @@ -405,9 +400,9 @@ public: getDWARFAddressSpace(unsigned AddressSpace) const override { const unsigned DWARF_Private = 1; const unsigned DWARF_Local = 2; - if (AddressSpace == Private) { + if (AddressSpace == llvm::AMDGPUAS::PRIVATE_ADDRESS) { return DWARF_Private; - } else if (AddressSpace == Local) { + } else if (AddressSpace == llvm::AMDGPUAS::LOCAL_ADDRESS) { return DWARF_Local; } else { return std::nullopt; diff --git a/clang/lib/Basic/Targets/NVPTX.cpp b/clang/lib/Basic/Targets/NVPTX.cpp index 3a4a75b0348f2094c84f4343bc98fccb69e51887..5c601812f617596526cc6d2a0e22cbc23bb5e05d 100644 --- a/clang/lib/Basic/Targets/NVPTX.cpp +++ b/clang/lib/Basic/Targets/NVPTX.cpp @@ -262,11 +262,14 @@ void NVPTXTargetInfo::getTargetDefines(const LangOptions &Opts, case CudaArch::SM_89: return "890"; case CudaArch::SM_90: + case CudaArch::SM_90a: return "900"; } llvm_unreachable("unhandled CudaArch"); }(); Builder.defineMacro("__CUDA_ARCH__", CUDAArchCode); + if (GPU == CudaArch::SM_90a) + Builder.defineMacro("__CUDA_ARCH_FEAT_SM90_ALL", "1"); } } diff --git a/clang/lib/Basic/Targets/OSTargets.cpp b/clang/lib/Basic/Targets/OSTargets.cpp index 627bc912fa2310464745d1555e49f64874cde1f2..899aefa6173acfe131a436852f4d4e2661768b68 100644 --- a/clang/lib/Basic/Targets/OSTargets.cpp +++ b/clang/lib/Basic/Targets/OSTargets.cpp @@ -224,6 +224,9 @@ static void addVisualCDefines(const LangOptions &Opts, MacroBuilder &Builder) { else if (Opts.CPlusPlus14) Builder.defineMacro("_MSVC_LANG", "201402L"); } + + if (Opts.isCompatibleWithMSVC(LangOptions::MSVC2022_3)) + Builder.defineMacro("_MSVC_CONSTEXPR_ATTRIBUTE"); } if (Opts.MicrosoftExt) { diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp index 13f934e9947212c67535d7fd4cb0e8434e7611e3..45d23022b5306b3c5e5ec41d637520dccc0cde72 100644 --- a/clang/lib/Basic/Targets/RISCV.cpp +++ b/clang/lib/Basic/Targets/RISCV.cpp @@ -131,7 +131,7 @@ static unsigned getVersionValue(unsigned MajorVersion, unsigned MinorVersion) { void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts, MacroBuilder &Builder) const { Builder.defineMacro("__riscv"); - bool Is64Bit = getTriple().getArch() == llvm::Triple::riscv64; + bool Is64Bit = getTriple().isRISCV64(); Builder.defineMacro("__riscv_xlen", Is64Bit ? "64" : "32"); StringRef CodeModel = getTargetOpts().CodeModel; unsigned FLen = ISAInfo->getFLen(); @@ -281,7 +281,7 @@ bool RISCVTargetInfo::initFeatureMap( unsigned XLen = 32; - if (getTriple().getArch() == llvm::Triple::riscv64) { + if (getTriple().isRISCV64()) { Features["64bit"] = true; XLen = 64; } else { @@ -304,11 +304,18 @@ bool RISCVTargetInfo::initFeatureMap( // RISCVISAInfo makes implications for ISA features std::vector ImpliedFeatures = (*ParseResult)->toFeatureVector(); - // Add non-ISA features like `relax` and `save-restore` back - for (const std::string &Feature : NewFeaturesVec) - if (!llvm::is_contained(ImpliedFeatures, Feature)) - ImpliedFeatures.push_back(Feature); + // parseFeatures normalizes the feature set by dropping any explicit + // negatives, and non-extension features. We need to preserve the later + // for correctness and want to preserve the former for consistency. + for (auto &Feature : NewFeaturesVec) { + StringRef ExtName = Feature; + assert(ExtName.size() > 1 && (ExtName[0] == '+' || ExtName[0] == '-')); + ExtName = ExtName.drop_front(1); // Drop '+' or '-' + if (!llvm::is_contained(ImpliedFeatures, ("+" + ExtName).str()) && + !llvm::is_contained(ImpliedFeatures, ("-" + ExtName).str())) + ImpliedFeatures.push_back(Feature); + } return TargetInfo::initFeatureMap(Features, Diags, CPU, ImpliedFeatures); } @@ -336,7 +343,7 @@ RISCVTargetInfo::getVScaleRange(const LangOptions &LangOpts) const { /// Return true if has this feature, need to sync with handleTargetFeatures. bool RISCVTargetInfo::hasFeature(StringRef Feature) const { - bool Is64Bit = getTriple().getArch() == llvm::Triple::riscv64; + bool Is64Bit = getTriple().isRISCV64(); auto Result = llvm::StringSwitch>(Feature) .Case("riscv", true) .Case("riscv32", !Is64Bit) @@ -347,10 +354,7 @@ bool RISCVTargetInfo::hasFeature(StringRef Feature) const { if (Result) return *Result; - if (ISAInfo->isSupportedExtensionFeature(Feature)) - return ISAInfo->hasExtension(Feature); - - return false; + return ISAInfo->hasExtension(Feature); } /// Perform initialization based on the user configured set of features. diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index 8c666e2cb463c6ccfee6259d1339d693893f9a07..77455c075cab0d9dadd52756f3a47bd62d55835d 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -982,7 +982,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline( getInstrProfOptions(CodeGenOpts, LangOpts)) PB.registerPipelineStartEPCallback( [Options](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(InstrProfiling(*Options, false)); + MPM.addPass(InstrProfilingLoweringPass(*Options, false)); }); // TODO: Consider passing the MemoryProfileOutput to the pass builder via diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp index 379c833af32a2de87e57b5057cafe8026d59220a..52e6ddb7d6afb05c217ade3f6c49016550a5e0dd 100644 --- a/clang/lib/CodeGen/CGAtomic.cpp +++ b/clang/lib/CodeGen/CGAtomic.cpp @@ -507,9 +507,11 @@ static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder, default: llvm_unreachable("Unexpected min/max operation"); case AtomicExpr::AO__atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT; break; case AtomicExpr::AO__atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT; break; } @@ -544,7 +546,9 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, FailureOrder, Size, Order, Scope); return; case AtomicExpr::AO__atomic_compare_exchange: - case AtomicExpr::AO__atomic_compare_exchange_n: { + case AtomicExpr::AO__atomic_compare_exchange_n: + case AtomicExpr::AO__scoped_atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: { if (llvm::ConstantInt *IsWeakC = dyn_cast(IsWeak)) { emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, Val1, Val2, FailureOrder, Size, Order, Scope); @@ -577,7 +581,9 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__opencl_atomic_load: case AtomicExpr::AO__hip_atomic_load: case AtomicExpr::AO__atomic_load_n: - case AtomicExpr::AO__atomic_load: { + case AtomicExpr::AO__atomic_load: + case AtomicExpr::AO__scoped_atomic_load_n: + case AtomicExpr::AO__scoped_atomic_load: { llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); Load->setAtomic(Order, Scope); Load->setVolatile(E->isVolatile()); @@ -589,7 +595,9 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__opencl_atomic_store: case AtomicExpr::AO__hip_atomic_store: case AtomicExpr::AO__atomic_store: - case AtomicExpr::AO__atomic_store_n: { + case AtomicExpr::AO__atomic_store_n: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_store_n: { llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); Store->setAtomic(Order, Scope); @@ -602,10 +610,13 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__opencl_atomic_exchange: case AtomicExpr::AO__atomic_exchange_n: case AtomicExpr::AO__atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange_n: + case AtomicExpr::AO__scoped_atomic_exchange: Op = llvm::AtomicRMWInst::Xchg; break; case AtomicExpr::AO__atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_add_fetch: PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd : llvm::Instruction::Add; [[fallthrough]]; @@ -613,11 +624,13 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__hip_atomic_fetch_add: case AtomicExpr::AO__opencl_atomic_fetch_add: case AtomicExpr::AO__atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_add: Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd : llvm::AtomicRMWInst::Add; break; case AtomicExpr::AO__atomic_sub_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub : llvm::Instruction::Sub; [[fallthrough]]; @@ -625,17 +638,20 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__hip_atomic_fetch_sub: case AtomicExpr::AO__opencl_atomic_fetch_sub: case AtomicExpr::AO__atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_fetch_sub: Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub : llvm::AtomicRMWInst::Sub; break; case AtomicExpr::AO__atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: PostOpMinMax = true; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_min: case AtomicExpr::AO__hip_atomic_fetch_min: case AtomicExpr::AO__opencl_atomic_fetch_min: case AtomicExpr::AO__atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_min: Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FMin : (E->getValueType()->isSignedIntegerType() @@ -644,12 +660,14 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, break; case AtomicExpr::AO__atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: PostOpMinMax = true; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_max: case AtomicExpr::AO__hip_atomic_fetch_max: case AtomicExpr::AO__opencl_atomic_fetch_max: case AtomicExpr::AO__atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_max: Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FMax : (E->getValueType()->isSignedIntegerType() @@ -658,40 +676,48 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, break; case AtomicExpr::AO__atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_and_fetch: PostOp = llvm::Instruction::And; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_and: case AtomicExpr::AO__hip_atomic_fetch_and: case AtomicExpr::AO__opencl_atomic_fetch_and: case AtomicExpr::AO__atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_and: Op = llvm::AtomicRMWInst::And; break; case AtomicExpr::AO__atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: PostOp = llvm::Instruction::Or; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_or: case AtomicExpr::AO__hip_atomic_fetch_or: case AtomicExpr::AO__opencl_atomic_fetch_or: case AtomicExpr::AO__atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_or: Op = llvm::AtomicRMWInst::Or; break; case AtomicExpr::AO__atomic_xor_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: PostOp = llvm::Instruction::Xor; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_xor: case AtomicExpr::AO__hip_atomic_fetch_xor: case AtomicExpr::AO__opencl_atomic_fetch_xor: case AtomicExpr::AO__atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_fetch_xor: Op = llvm::AtomicRMWInst::Xor; break; case AtomicExpr::AO__atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: PostOp = llvm::Instruction::And; // the NOT is special cased below [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_nand: case AtomicExpr::AO__atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_fetch_nand: Op = llvm::AtomicRMWInst::Nand; break; } @@ -711,7 +737,8 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, else if (PostOp) Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI, LoadVal1); - if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) + if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch || + E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch) Result = CGF.Builder.CreateNot(Result); CGF.Builder.CreateStore(Result, Dest); } @@ -861,20 +888,24 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { llvm_unreachable("Already handled above with EmitAtomicInit!"); case AtomicExpr::AO__atomic_load_n: + case AtomicExpr::AO__scoped_atomic_load_n: case AtomicExpr::AO__c11_atomic_load: case AtomicExpr::AO__opencl_atomic_load: case AtomicExpr::AO__hip_atomic_load: break; case AtomicExpr::AO__atomic_load: + case AtomicExpr::AO__scoped_atomic_load: Dest = EmitPointerWithAlignment(E->getVal1()); break; case AtomicExpr::AO__atomic_store: + case AtomicExpr::AO__scoped_atomic_store: Val1 = EmitPointerWithAlignment(E->getVal1()); break; case AtomicExpr::AO__atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange: Val1 = EmitPointerWithAlignment(E->getVal1()); Dest = EmitPointerWithAlignment(E->getVal2()); break; @@ -887,14 +918,19 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__hip_atomic_compare_exchange_strong: case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: + case AtomicExpr::AO__scoped_atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: Val1 = EmitPointerWithAlignment(E->getVal1()); - if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) + if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange || + E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange) Val2 = EmitPointerWithAlignment(E->getVal2()); else Val2 = EmitValToTemp(*this, E->getVal2()); OrderFail = EmitScalarExpr(E->getOrderFail()); if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || - E->getOp() == AtomicExpr::AO__atomic_compare_exchange) + E->getOp() == AtomicExpr::AO__atomic_compare_exchange || + E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n || + E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange) IsWeak = EmitScalarExpr(E->getWeak()); break; @@ -934,6 +970,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__opencl_atomic_fetch_min: case AtomicExpr::AO__hip_atomic_fetch_max: case AtomicExpr::AO__hip_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: ShouldCastToIntPtrTy = !MemTy->isFloatingType(); [[fallthrough]]; @@ -963,6 +1007,16 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__opencl_atomic_fetch_xor: case AtomicExpr::AO__opencl_atomic_store: case AtomicExpr::AO__opencl_atomic_exchange: + case AtomicExpr::AO__scoped_atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: + case AtomicExpr::AO__scoped_atomic_store_n: + case AtomicExpr::AO__scoped_atomic_exchange_n: Val1 = EmitValToTemp(*this, E->getVal1()); break; } @@ -1039,6 +1093,22 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__opencl_atomic_fetch_or: case AtomicExpr::AO__opencl_atomic_fetch_sub: case AtomicExpr::AO__opencl_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: // For these, only library calls for certain sizes exist. UseOptimizedLibcall = true; break; @@ -1047,6 +1117,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__atomic_store: case AtomicExpr::AO__atomic_exchange: case AtomicExpr::AO__atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_load: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange: // Use the generic version if we don't know that the operand will be // suitably aligned for the optimized version. if (Misaligned) @@ -1071,6 +1145,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__opencl_atomic_exchange: case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: + case AtomicExpr::AO__scoped_atomic_load_n: + case AtomicExpr::AO__scoped_atomic_store_n: + case AtomicExpr::AO__scoped_atomic_exchange_n: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: // Only use optimized library calls for sizes for which they exist. // FIXME: Size == 16 optimized library functions exist too. if (Size == 1 || Size == 2 || Size == 4 || Size == 8) @@ -1131,6 +1209,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__hip_atomic_compare_exchange_strong: case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: + case AtomicExpr::AO__scoped_atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: LibCallName = "__atomic_compare_exchange"; RetTy = getContext().BoolTy; HaveRetTy = true; @@ -1150,6 +1230,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__c11_atomic_exchange: case AtomicExpr::AO__hip_atomic_exchange: case AtomicExpr::AO__opencl_atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange_n: LibCallName = "__atomic_exchange"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); @@ -1161,6 +1243,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__c11_atomic_store: case AtomicExpr::AO__hip_atomic_store: case AtomicExpr::AO__opencl_atomic_store: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_store_n: LibCallName = "__atomic_store"; RetTy = getContext().VoidTy; HaveRetTy = true; @@ -1174,17 +1258,21 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__c11_atomic_load: case AtomicExpr::AO__hip_atomic_load: case AtomicExpr::AO__opencl_atomic_load: + case AtomicExpr::AO__scoped_atomic_load: + case AtomicExpr::AO__scoped_atomic_load_n: LibCallName = "__atomic_load"; break; // T __atomic_add_fetch_N(T *mem, T val, int order) // T __atomic_fetch_add_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_add_fetch: PostOp = llvm::Instruction::Add; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_add: case AtomicExpr::AO__c11_atomic_fetch_add: case AtomicExpr::AO__hip_atomic_fetch_add: case AtomicExpr::AO__opencl_atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_add: LibCallName = "__atomic_fetch_add"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), LoweredMemTy, E->getExprLoc(), TInfo.Width); @@ -1192,12 +1280,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_and_fetch_N(T *mem, T val, int order) // T __atomic_fetch_and_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_and_fetch: PostOp = llvm::Instruction::And; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_and: case AtomicExpr::AO__c11_atomic_fetch_and: case AtomicExpr::AO__hip_atomic_fetch_and: case AtomicExpr::AO__opencl_atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_and: LibCallName = "__atomic_fetch_and"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); @@ -1205,12 +1295,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_or_fetch_N(T *mem, T val, int order) // T __atomic_fetch_or_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: PostOp = llvm::Instruction::Or; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_or: case AtomicExpr::AO__c11_atomic_fetch_or: case AtomicExpr::AO__hip_atomic_fetch_or: case AtomicExpr::AO__opencl_atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_or: LibCallName = "__atomic_fetch_or"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); @@ -1218,12 +1310,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_sub_fetch_N(T *mem, T val, int order) // T __atomic_fetch_sub_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_sub_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: PostOp = llvm::Instruction::Sub; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_sub: case AtomicExpr::AO__c11_atomic_fetch_sub: case AtomicExpr::AO__hip_atomic_fetch_sub: case AtomicExpr::AO__opencl_atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_fetch_sub: LibCallName = "__atomic_fetch_sub"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), LoweredMemTy, E->getExprLoc(), TInfo.Width); @@ -1231,21 +1325,25 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_xor_fetch_N(T *mem, T val, int order) // T __atomic_fetch_xor_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_xor_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: PostOp = llvm::Instruction::Xor; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_xor: case AtomicExpr::AO__c11_atomic_fetch_xor: case AtomicExpr::AO__hip_atomic_fetch_xor: case AtomicExpr::AO__opencl_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_fetch_xor: LibCallName = "__atomic_fetch_xor"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); break; case AtomicExpr::AO__atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: PostOpMinMax = true; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_min: case AtomicExpr::AO__c11_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_min: case AtomicExpr::AO__hip_atomic_fetch_min: case AtomicExpr::AO__opencl_atomic_fetch_min: LibCallName = E->getValueType()->isSignedIntegerType() @@ -1255,12 +1353,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { LoweredMemTy, E->getExprLoc(), TInfo.Width); break; case AtomicExpr::AO__atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: PostOpMinMax = true; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_max: case AtomicExpr::AO__c11_atomic_fetch_max: case AtomicExpr::AO__hip_atomic_fetch_max: case AtomicExpr::AO__opencl_atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_max: LibCallName = E->getValueType()->isSignedIntegerType() ? "__atomic_fetch_max" : "__atomic_fetch_umax"; @@ -1270,10 +1370,12 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_nand_fetch_N(T *mem, T val, int order) // T __atomic_fetch_nand_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: PostOp = llvm::Instruction::And; // the NOT is special cased below [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_nand: case AtomicExpr::AO__c11_atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_fetch_nand: LibCallName = "__atomic_fetch_nand"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); @@ -1330,7 +1432,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); } - if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) + if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch || + E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch) ResVal = Builder.CreateNot(ResVal); Builder.CreateStore(ResVal, Dest.withElementType(ResVal->getType())); @@ -1347,12 +1450,16 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { E->getOp() == AtomicExpr::AO__opencl_atomic_store || E->getOp() == AtomicExpr::AO__hip_atomic_store || E->getOp() == AtomicExpr::AO__atomic_store || - E->getOp() == AtomicExpr::AO__atomic_store_n; + E->getOp() == AtomicExpr::AO__atomic_store_n || + E->getOp() == AtomicExpr::AO__scoped_atomic_store || + E->getOp() == AtomicExpr::AO__scoped_atomic_store_n; bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || E->getOp() == AtomicExpr::AO__opencl_atomic_load || E->getOp() == AtomicExpr::AO__hip_atomic_load || E->getOp() == AtomicExpr::AO__atomic_load || - E->getOp() == AtomicExpr::AO__atomic_load_n; + E->getOp() == AtomicExpr::AO__atomic_load_n || + E->getOp() == AtomicExpr::AO__scoped_atomic_load || + E->getOp() == AtomicExpr::AO__scoped_atomic_load_n; if (isa(Order)) { auto ord = cast(Order)->getZExtValue(); diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 0d8b3e4aaad4705994ea6d0fdc9ebbcd090671ee..83d0a72aac5495f55d960f066bfff6bc4a201324 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -10211,6 +10211,22 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, switch (BuiltinID) { default: return nullptr; + + case SVE::BI__builtin_sve_svreinterpret_b: { + auto SVCountTy = + llvm::TargetExtType::get(getLLVMContext(), "aarch64.svcount"); + Function *CastFromSVCountF = + CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy); + return Builder.CreateCall(CastFromSVCountF, Ops[0]); + } + case SVE::BI__builtin_sve_svreinterpret_c: { + auto SVCountTy = + llvm::TargetExtType::get(getLLVMContext(), "aarch64.svcount"); + Function *CastToSVCountF = + CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy); + return Builder.CreateCall(CastToSVCountF, Ops[0]); + } + case SVE::BI__builtin_sve_svpsel_lane_b8: case SVE::BI__builtin_sve_svpsel_lane_b16: case SVE::BI__builtin_sve_svpsel_lane_b32: diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp index 66147f656071f5383a54effe15652293b49f49b2..520b0c4f117673f76b93279d7bdaa0f26a21784f 100644 --- a/clang/lib/CodeGen/CGCUDANV.cpp +++ b/clang/lib/CodeGen/CGCUDANV.cpp @@ -1132,26 +1132,39 @@ void CGNVCUDARuntime::createOffloadingEntries() { for (KernelInfo &I : EmittedKernels) llvm::offloading::emitOffloadingEntry( M, KernelHandles[I.Kernel->getName()], - getDeviceSideName(cast(I.D)), 0, - DeviceVarFlags::OffloadGlobalEntry, Section); + getDeviceSideName(cast(I.D)), /*Flags=*/0, /*Data=*/0, + llvm::offloading::OffloadGlobalEntry, Section); for (VarInfo &I : DeviceVars) { uint64_t VarSize = CGM.getDataLayout().getTypeAllocSize(I.Var->getValueType()); + int32_t Flags = + (I.Flags.isExtern() + ? static_cast(llvm::offloading::OffloadGlobalExtern) + : 0) | + (I.Flags.isConstant() + ? static_cast(llvm::offloading::OffloadGlobalConstant) + : 0) | + (I.Flags.isNormalized() + ? static_cast(llvm::offloading::OffloadGlobalNormalized) + : 0); if (I.Flags.getKind() == DeviceVarFlags::Variable) { llvm::offloading::emitOffloadingEntry( M, I.Var, getDeviceSideName(I.D), VarSize, - I.Flags.isManaged() ? DeviceVarFlags::OffloadGlobalManagedEntry - : DeviceVarFlags::OffloadGlobalEntry, - Section); + (I.Flags.isManaged() ? llvm::offloading::OffloadGlobalManagedEntry + : llvm::offloading::OffloadGlobalEntry) | + Flags, + /*Data=*/0, Section); } else if (I.Flags.getKind() == DeviceVarFlags::Surface) { llvm::offloading::emitOffloadingEntry( M, I.Var, getDeviceSideName(I.D), VarSize, - DeviceVarFlags::OffloadGlobalSurfaceEntry, Section); + llvm::offloading::OffloadGlobalSurfaceEntry | Flags, + I.Flags.getSurfTexType(), Section); } else if (I.Flags.getKind() == DeviceVarFlags::Texture) { llvm::offloading::emitOffloadingEntry( M, I.Var, getDeviceSideName(I.D), VarSize, - DeviceVarFlags::OffloadGlobalTextureEntry, Section); + llvm::offloading::OffloadGlobalTextureEntry | Flags, + I.Flags.getSurfTexType(), Section); } } } diff --git a/clang/lib/CodeGen/CGCUDARuntime.h b/clang/lib/CodeGen/CGCUDARuntime.h index 9a9c6d26cc63c40c4d74c39337afb77563fa4daa..c7af8f1cf0fe95ee7d1e7ded0cec6507198f6967 100644 --- a/clang/lib/CodeGen/CGCUDARuntime.h +++ b/clang/lib/CodeGen/CGCUDARuntime.h @@ -17,6 +17,7 @@ #include "clang/AST/GlobalDecl.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Frontend/Offloading/Utility.h" #include "llvm/IR/GlobalValue.h" namespace llvm { @@ -52,19 +53,6 @@ public: Texture, // Builtin texture }; - /// The kind flag for an offloading entry. - enum OffloadEntryKindFlag : uint32_t { - /// Mark the entry as a global entry. This indicates the presense of a - /// kernel if the size field is zero and a variable otherwise. - OffloadGlobalEntry = 0x0, - /// Mark the entry as a managed global variable. - OffloadGlobalManagedEntry = 0x1, - /// Mark the entry as a surface variable. - OffloadGlobalSurfaceEntry = 0x2, - /// Mark the entry as a texture variable. - OffloadGlobalTextureEntry = 0x3, - }; - private: unsigned Kind : 2; unsigned Extern : 1; diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h index aee86a3242fd3f4468740464fc13f09be9e0d06b..1c0d15dc932ad80f8ca6a684bcbf3380857c0f59 100644 --- a/clang/lib/CodeGen/CGCall.h +++ b/clang/lib/CodeGen/CGCall.h @@ -20,6 +20,7 @@ #include "clang/AST/CanonicalType.h" #include "clang/AST/GlobalDecl.h" #include "clang/AST/Type.h" +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/IR/Value.h" namespace llvm { @@ -406,15 +407,13 @@ enum class FnInfoOpts { }; inline FnInfoOpts operator|(FnInfoOpts A, FnInfoOpts B) { - return static_cast( - static_cast>(A) | - static_cast>(B)); + return static_cast(llvm::to_underlying(A) | + llvm::to_underlying(B)); } inline FnInfoOpts operator&(FnInfoOpts A, FnInfoOpts B) { - return static_cast( - static_cast>(A) & - static_cast>(B)); + return static_cast(llvm::to_underlying(A) & + llvm::to_underlying(B)); } inline FnInfoOpts operator|=(FnInfoOpts A, FnInfoOpts B) { diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp index f3cbd1d0451ebe4f8d15b24306fb685d763430d2..e532794b71bdb4a9beed87128d409b47dd06263a 100644 --- a/clang/lib/CodeGen/CGExprComplex.cpp +++ b/clang/lib/CodeGen/CGExprComplex.cpp @@ -275,6 +275,10 @@ public: ComplexPairTy EmitBinSub(const BinOpInfo &Op); ComplexPairTy EmitBinMul(const BinOpInfo &Op); ComplexPairTy EmitBinDiv(const BinOpInfo &Op); + ComplexPairTy EmitAlgebraicDiv(llvm::Value *A, llvm::Value *B, llvm::Value *C, + llvm::Value *D); + ComplexPairTy EmitRangeReductionDiv(llvm::Value *A, llvm::Value *B, + llvm::Value *C, llvm::Value *D); ComplexPairTy EmitComplexBinOpLibCall(StringRef LibCallName, const BinOpInfo &Op); @@ -781,6 +785,10 @@ ComplexPairTy ComplexExprEmitter::EmitBinMul(const BinOpInfo &Op) { ResR = Builder.CreateFSub(AC, BD, "mul_r"); ResI = Builder.CreateFAdd(AD, BC, "mul_i"); + if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Limited || + Op.FPFeatures.getComplexRange() == LangOptions::CX_Fortran) + return ComplexPairTy(ResR, ResI); + // Emit the test for the real part becoming NaN and create a branch to // handle it. We test for NaN by comparing the number to itself. Value *IsRNaN = Builder.CreateFCmpUNO(ResR, ResR, "isnan_cmp"); @@ -846,23 +854,139 @@ ComplexPairTy ComplexExprEmitter::EmitBinMul(const BinOpInfo &Op) { return ComplexPairTy(ResR, ResI); } +ComplexPairTy ComplexExprEmitter::EmitAlgebraicDiv(llvm::Value *LHSr, + llvm::Value *LHSi, + llvm::Value *RHSr, + llvm::Value *RHSi) { + // (a+ib) / (c+id) = ((ac+bd)/(cc+dd)) + i((bc-ad)/(cc+dd)) + llvm::Value *DSTr, *DSTi; + + llvm::Value *AC = Builder.CreateFMul(LHSr, RHSr); // a*c + llvm::Value *BD = Builder.CreateFMul(LHSi, RHSi); // b*d + llvm::Value *ACpBD = Builder.CreateFAdd(AC, BD); // ac+bd + + llvm::Value *CC = Builder.CreateFMul(RHSr, RHSr); // c*c + llvm::Value *DD = Builder.CreateFMul(RHSi, RHSi); // d*d + llvm::Value *CCpDD = Builder.CreateFAdd(CC, DD); // cc+dd + + llvm::Value *BC = Builder.CreateFMul(LHSi, RHSr); // b*c + llvm::Value *AD = Builder.CreateFMul(LHSr, RHSi); // a*d + llvm::Value *BCmAD = Builder.CreateFSub(BC, AD); // bc-ad + + DSTr = Builder.CreateFDiv(ACpBD, CCpDD); + DSTi = Builder.CreateFDiv(BCmAD, CCpDD); + return ComplexPairTy(DSTr, DSTi); +} + +// EmitFAbs - Emit a call to @llvm.fabs. +static llvm::Value *EmitllvmFAbs(CodeGenFunction &CGF, llvm::Value *Value) { + llvm::Function *Func = + CGF.CGM.getIntrinsic(llvm::Intrinsic::fabs, Value->getType()); + llvm::Value *Call = CGF.Builder.CreateCall(Func, Value); + return Call; +} + +// EmitRangeReductionDiv - Implements Smith's algorithm for complex division. +// SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8 (1962). +ComplexPairTy ComplexExprEmitter::EmitRangeReductionDiv(llvm::Value *LHSr, + llvm::Value *LHSi, + llvm::Value *RHSr, + llvm::Value *RHSi) { + // (a + ib) / (c + id) = (e + if) + llvm::Value *FAbsRHSr = EmitllvmFAbs(CGF, RHSr); // |c| + llvm::Value *FAbsRHSi = EmitllvmFAbs(CGF, RHSi); // |d| + // |c| >= |d| + llvm::Value *IsR = Builder.CreateFCmpUGT(FAbsRHSr, FAbsRHSi, "abs_cmp"); + + llvm::BasicBlock *TrueBB = + CGF.createBasicBlock("abs_rhsr_greater_or_equal_abs_rhsi"); + llvm::BasicBlock *FalseBB = + CGF.createBasicBlock("abs_rhsr_less_than_abs_rhsi"); + llvm::BasicBlock *ContBB = CGF.createBasicBlock("complex_div"); + Builder.CreateCondBr(IsR, TrueBB, FalseBB); + + CGF.EmitBlock(TrueBB); + // abs(c) >= abs(d) + // r = d/c + // tmp = c + rd + // e = (a + br)/tmp + // f = (b - ar)/tmp + llvm::Value *DdC = Builder.CreateFDiv(RHSi, RHSr); // r=d/c + + llvm::Value *RD = Builder.CreateFMul(DdC, RHSi); // rd + llvm::Value *CpRD = Builder.CreateFAdd(RHSr, RD); // tmp=c+rd + + llvm::Value *T3 = Builder.CreateFMul(LHSi, DdC); // br + llvm::Value *T4 = Builder.CreateFAdd(LHSr, T3); // a+br + llvm::Value *DSTTr = Builder.CreateFDiv(T4, CpRD); // (a+br)/tmp + + llvm::Value *T5 = Builder.CreateFMul(LHSr, DdC); // ar + llvm::Value *T6 = Builder.CreateFSub(LHSi, T5); // b-ar + llvm::Value *DSTTi = Builder.CreateFDiv(T6, CpRD); // (b-ar)/tmp + Builder.CreateBr(ContBB); + + CGF.EmitBlock(FalseBB); + // abs(c) < abs(d) + // r = c/d + // tmp = d + rc + // e = (ar + b)/tmp + // f = (br - a)/tmp + llvm::Value *CdD = Builder.CreateFDiv(RHSr, RHSi); // r=c/d + + llvm::Value *RC = Builder.CreateFMul(CdD, RHSr); // rc + llvm::Value *DpRC = Builder.CreateFAdd(RHSi, RC); // tmp=d+rc + + llvm::Value *T7 = Builder.CreateFMul(LHSr, RC); // ar + llvm::Value *T8 = Builder.CreateFAdd(T7, LHSi); // ar+b + llvm::Value *DSTFr = Builder.CreateFDiv(T8, DpRC); // (ar+b)/tmp + + llvm::Value *T9 = Builder.CreateFMul(LHSi, CdD); // br + llvm::Value *T10 = Builder.CreateFSub(T9, LHSr); // br-a + llvm::Value *DSTFi = Builder.CreateFDiv(T10, DpRC); // (br-a)/tmp + Builder.CreateBr(ContBB); + + // Phi together the computation paths. + CGF.EmitBlock(ContBB); + llvm::PHINode *VALr = Builder.CreatePHI(DSTTr->getType(), 2); + VALr->addIncoming(DSTTr, TrueBB); + VALr->addIncoming(DSTFr, FalseBB); + llvm::PHINode *VALi = Builder.CreatePHI(DSTTi->getType(), 2); + VALi->addIncoming(DSTTi, TrueBB); + VALi->addIncoming(DSTFi, FalseBB); + return ComplexPairTy(VALr, VALi); +} + // See C11 Annex G.5.1 for the semantics of multiplicative operators on complex // typed values. ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) { llvm::Value *LHSr = Op.LHS.first, *LHSi = Op.LHS.second; llvm::Value *RHSr = Op.RHS.first, *RHSi = Op.RHS.second; - llvm::Value *DSTr, *DSTi; if (LHSr->getType()->isFloatingPointTy()) { - // If we have a complex operand on the RHS and FastMath is not allowed, we - // delegate to a libcall to handle all of the complexities and minimize - // underflow/overflow cases. When FastMath is allowed we construct the - // divide inline using the same algorithm as for integer operands. - // - // FIXME: We would be able to avoid the libcall in many places if we - // supported imaginary types in addition to complex types. CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op.FPFeatures); - if (RHSi && !CGF.getLangOpts().FastMath) { + if (!RHSi) { + assert(LHSi && "Can have at most one non-complex operand!"); + + DSTr = Builder.CreateFDiv(LHSr, RHSr); + DSTi = Builder.CreateFDiv(LHSi, RHSr); + return ComplexPairTy(DSTr, DSTi); + } + llvm::Value *OrigLHSi = LHSi; + if (!LHSi) + LHSi = llvm::Constant::getNullValue(RHSi->getType()); + if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Fortran) + return EmitRangeReductionDiv(LHSr, LHSi, RHSr, RHSi); + else if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Limited) + return EmitAlgebraicDiv(LHSr, LHSi, RHSr, RHSi); + else if (!CGF.getLangOpts().FastMath) { + LHSi = OrigLHSi; + // If we have a complex operand on the RHS and FastMath is not allowed, we + // delegate to a libcall to handle all of the complexities and minimize + // underflow/overflow cases. When FastMath is allowed we construct the + // divide inline using the same algorithm as for integer operands. + // + // FIXME: We would be able to avoid the libcall in many places if we + // supported imaginary types in addition to complex types. BinOpInfo LibCallOp = Op; // If LHS was a real, supply a null imaginary part. if (!LHSi) @@ -884,30 +1008,8 @@ ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) { case llvm::Type::FP128TyID: return EmitComplexBinOpLibCall("__divtc3", LibCallOp); } - } else if (RHSi) { - if (!LHSi) - LHSi = llvm::Constant::getNullValue(RHSi->getType()); - - // (a+ib) / (c+id) = ((ac+bd)/(cc+dd)) + i((bc-ad)/(cc+dd)) - llvm::Value *AC = Builder.CreateFMul(LHSr, RHSr); // a*c - llvm::Value *BD = Builder.CreateFMul(LHSi, RHSi); // b*d - llvm::Value *ACpBD = Builder.CreateFAdd(AC, BD); // ac+bd - - llvm::Value *CC = Builder.CreateFMul(RHSr, RHSr); // c*c - llvm::Value *DD = Builder.CreateFMul(RHSi, RHSi); // d*d - llvm::Value *CCpDD = Builder.CreateFAdd(CC, DD); // cc+dd - - llvm::Value *BC = Builder.CreateFMul(LHSi, RHSr); // b*c - llvm::Value *AD = Builder.CreateFMul(LHSr, RHSi); // a*d - llvm::Value *BCmAD = Builder.CreateFSub(BC, AD); // bc-ad - - DSTr = Builder.CreateFDiv(ACpBD, CCpDD); - DSTi = Builder.CreateFDiv(BCmAD, CCpDD); } else { - assert(LHSi && "Can have at most one non-complex operand!"); - - DSTr = Builder.CreateFDiv(LHSr, RHSr); - DSTi = Builder.CreateFDiv(LHSi, RHSr); + return EmitAlgebraicDiv(LHSr, LHSi, RHSr, RHSi); } } else { assert(Op.LHS.second && Op.RHS.second && diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp index c239bc17ef267e86381233cebfbbee9eeb59864a..3e8a40e7540bef7060a4f5fe124250483e684c46 100644 --- a/clang/lib/CodeGen/CGHLSLRuntime.cpp +++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp @@ -184,7 +184,8 @@ void CGHLSLRuntime::finishCodeGen() { : llvm::hlsl::ResourceKind::TBuffer; std::string TyName = Buf.Name.str() + (Buf.IsCBuffer ? ".cb." : ".tb.") + "ty"; - addBufferResourceAnnotation(GV, TyName, RC, RK, Buf.Binding); + addBufferResourceAnnotation(GV, TyName, RC, RK, /*IsROV=*/false, + Buf.Binding); } } @@ -196,6 +197,7 @@ void CGHLSLRuntime::addBufferResourceAnnotation(llvm::GlobalVariable *GV, llvm::StringRef TyName, llvm::hlsl::ResourceClass RC, llvm::hlsl::ResourceKind RK, + bool IsROV, BufferResBinding &Binding) { llvm::Module &M = CGM.getModule(); @@ -219,7 +221,7 @@ void CGHLSLRuntime::addBufferResourceAnnotation(llvm::GlobalVariable *GV, "ResourceMD must have been set by the switch above."); llvm::hlsl::FrontendResource Res( - GV, TyName, RK, Binding.Reg.value_or(UINT_MAX), Binding.Space); + GV, TyName, RK, IsROV, Binding.Reg.value_or(UINT_MAX), Binding.Space); ResourceMD->addOperand(Res.getMetadata()); } @@ -236,10 +238,11 @@ void CGHLSLRuntime::annotateHLSLResource(const VarDecl *D, GlobalVariable *GV) { llvm::hlsl::ResourceClass RC = Attr->getResourceClass(); llvm::hlsl::ResourceKind RK = Attr->getResourceKind(); + bool IsROV = Attr->getIsROV(); QualType QT(Ty, 0); BufferResBinding Binding(D->getAttr()); - addBufferResourceAnnotation(GV, QT.getAsString(), RC, RK, Binding); + addBufferResourceAnnotation(GV, QT.getAsString(), RC, RK, IsROV, Binding); } CGHLSLRuntime::BufferResBinding::BufferResBinding( diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h index 67413fbd4a78e1a99801499f1f33692a1094d136..bb500cb5c979f27d4e85d5440ae68aa207e69f19 100644 --- a/clang/lib/CodeGen/CGHLSLRuntime.h +++ b/clang/lib/CodeGen/CGHLSLRuntime.h @@ -92,7 +92,7 @@ private: void addBufferResourceAnnotation(llvm::GlobalVariable *GV, llvm::StringRef TyName, llvm::hlsl::ResourceClass RC, - llvm::hlsl::ResourceKind RK, + llvm::hlsl::ResourceKind RK, bool IsROV, BufferResBinding &Binding); void addConstant(VarDecl *D, Buffer &CB); void addBufferDecls(const DeclContext *DC, Buffer &CB); diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 55648963df36ab97811fecb15502f5a3cc85f421..7f7e6f5306664446ccbfdbfbe40ec7de13f0a977 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -7389,7 +7389,14 @@ private: } else if (FieldIndex < PartialStruct.LowestElem.first) { PartialStruct.LowestElem = {FieldIndex, LowestElem}; } else if (FieldIndex > PartialStruct.HighestElem.first) { - PartialStruct.HighestElem = {FieldIndex, LowestElem}; + if (IsFinalArraySection) { + Address HB = + CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false) + .getAddress(CGF); + PartialStruct.HighestElem = {FieldIndex, HB}; + } else { + PartialStruct.HighestElem = {FieldIndex, LowestElem}; + } } } diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp index 293ccaa3413cdf372d72a02ff4a93f4c2a28b947..299ee1460b3db0ef895c681246a8cc7f8bd42fb2 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp @@ -3483,6 +3483,7 @@ void CGOpenMPRuntimeGPU::processRequiresDirective( case CudaArch::SM_87: case CudaArch::SM_89: case CudaArch::SM_90: + case CudaArch::SM_90a: case CudaArch::GFX600: case CudaArch::GFX601: case CudaArch::GFX602: diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 6a20723bf2bca0c85e198a257dbcdaf13534c27c..b931a81bc00871b6d1de5fc0919e64467324b6cb 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -6439,7 +6439,7 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( VD, E->getManglingNumber(), Out); APValue *Value = nullptr; - if (E->getStorageDuration() == SD_Static && VD && VD->evaluateValue()) { + if (E->getStorageDuration() == SD_Static && VD->evaluateValue()) { // If the initializer of the extending declaration is a constant // initializer, we should have a cached constant initializer for this // temporary. Note that this might have a different value from the value @@ -6454,8 +6454,7 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( !EvalResult.hasSideEffects()) Value = &EvalResult.Val; - LangAS AddrSpace = - VD ? GetGlobalVarAddressSpace(VD) : MaterializedType.getAddressSpace(); + LangAS AddrSpace = GetGlobalVarAddressSpace(VD); std::optional emitter; llvm::Constant *InitialValue = nullptr; diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp index 5906b14dd93cf088bf9c25443d70c9a531416a00..dc288bc3f6157ac7dd42068046abf0888a740f69 100644 --- a/clang/lib/CodeGen/CodeGenTBAA.cpp +++ b/clang/lib/CodeGen/CodeGenTBAA.cpp @@ -196,11 +196,14 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) { // Enum types are distinct types. In C++ they have "underlying types", // however they aren't related for TBAA. if (const EnumType *ETy = dyn_cast(Ty)) { + if (!Features.CPlusPlus) + return getTypeInfo(ETy->getDecl()->getIntegerType()); + // In C++ mode, types have linkage, so we can rely on the ODR and // on their mangled names, if they're external. // TODO: Is there a way to get a program-wide unique name for a // decl with local linkage or no linkage? - if (!Features.CPlusPlus || !ETy->getDecl()->isExternallyVisible()) + if (!ETy->getDecl()->isExternallyVisible()) return getChar(); SmallString<256> OutName; diff --git a/clang/lib/CodeGen/Targets/AMDGPU.cpp b/clang/lib/CodeGen/Targets/AMDGPU.cpp index b654e3f12af8d4918e96784083b7b278c1691ca5..03ac6b78598fc8477e6756b4c64f5936fd0daedb 100644 --- a/clang/lib/CodeGen/Targets/AMDGPU.cpp +++ b/clang/lib/CodeGen/Targets/AMDGPU.cpp @@ -471,20 +471,25 @@ AMDGPUTargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &LangOpts, std::string Name; switch (Scope) { case SyncScope::HIPSingleThread: + case SyncScope::SingleScope: Name = "singlethread"; break; case SyncScope::HIPWavefront: case SyncScope::OpenCLSubGroup: + case SyncScope::WavefrontScope: Name = "wavefront"; break; case SyncScope::HIPWorkgroup: case SyncScope::OpenCLWorkGroup: + case SyncScope::WorkgroupScope: Name = "workgroup"; break; case SyncScope::HIPAgent: case SyncScope::OpenCLDevice: + case SyncScope::DeviceScope: Name = "agent"; break; + case SyncScope::SystemScope: case SyncScope::HIPSystem: case SyncScope::OpenCLAllSVMDevices: Name = ""; diff --git a/clang/lib/CrossTU/CrossTranslationUnit.cpp b/clang/lib/CrossTU/CrossTranslationUnit.cpp index 540c22d078654c01e454d46a576823c74753b859..94c10e50d7d06489084633a9ccdb11fc3d793e1e 100644 --- a/clang/lib/CrossTU/CrossTranslationUnit.cpp +++ b/clang/lib/CrossTU/CrossTranslationUnit.cpp @@ -551,7 +551,7 @@ CrossTranslationUnitContext::ASTLoader::load(StringRef Identifier) { // Normalize by removing relative path components. llvm::sys::path::remove_dots(Path, /*remove_dot_dot*/ true, PathStyle); - if (Path.endswith(".ast")) + if (Path.ends_with(".ast")) return loadFromDump(Path); else return loadFromSource(Path); diff --git a/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp b/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp index 9b3d2571f29f628d85200ae43c9838164641dc0e..beca9586988b526876c0787ae9d0e62345268c4f 100644 --- a/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp +++ b/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp @@ -14,7 +14,6 @@ #include "llvm/Support/AlignOf.h" #include "llvm/Support/Errno.h" #include "llvm/Support/Error.h" -#include "llvm/Support/MathExtras.h" #include "llvm/Support/Path.h" #include #include @@ -25,6 +24,7 @@ #include #include +#include #include #include #include diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index e241706b9082ee74bc7a1219ac38d67c89d3741c..f392f6794f857e62df637602027a56e99ad2fb9e 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -1522,7 +1522,7 @@ bool Driver::getCrashDiagnosticFile(StringRef ReproCrashFilename, // (or /Library/Logs/DiagnosticReports for root) and has the filename pattern // clang-__.crash. path::home_directory(CrashDiagDir); - if (CrashDiagDir.startswith("/var/root")) + if (CrashDiagDir.starts_with("/var/root")) CrashDiagDir = "/"; path::append(CrashDiagDir, "Library/Logs/DiagnosticReports"); int PID = diff --git a/clang/lib/Driver/Multilib.cpp b/clang/lib/Driver/Multilib.cpp index 7681c1a3ce6756f6cfc00cc13ed56b796a830ac5..9c091bbfdabab122cec9bb3c1035ece1988b0fc4 100644 --- a/clang/lib/Driver/Multilib.cpp +++ b/clang/lib/Driver/Multilib.cpp @@ -95,7 +95,7 @@ MultilibSet &MultilibSet::FilterOut(FilterCallback F) { void MultilibSet::push_back(const Multilib &M) { Multilibs.push_back(M); } bool MultilibSet::select(const Multilib::flags_list &Flags, - llvm::SmallVector &Selected) const { + llvm::SmallVectorImpl &Selected) const { llvm::StringSet<> FlagSet(expandFlags(Flags)); Selected.clear(); diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index f02f7c841b91f0ba56dd41e377429397404fb9d3..f95f3227aba7d03bb4956b62f39569ce6ee49768 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -1294,6 +1294,9 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA, CmdArgs.push_back("-source-date-epoch"); CmdArgs.push_back(Args.MakeArgString(Epoch)); } + + Args.addOptInFlag(CmdArgs, options::OPT_fdefine_target_os_macros, + options::OPT_fno_define_target_os_macros); } // FIXME: Move to target hook. @@ -2657,6 +2660,35 @@ static void CollectArgsForIntegratedAssembler(Compilation &C, } } +static StringRef EnumComplexRangeToStr(LangOptions::ComplexRangeKind Range) { + StringRef RangeStr = ""; + switch (Range) { + case LangOptions::ComplexRangeKind::CX_Limited: + return "-fcx-limited-range"; + break; + case LangOptions::ComplexRangeKind::CX_Fortran: + return "-fcx-fortran-rules"; + break; + default: + return RangeStr; + break; + } +} + +static void EmitComplexRangeDiag(const Driver &D, + LangOptions::ComplexRangeKind Range1, + LangOptions::ComplexRangeKind Range2) { + if (Range1 != LangOptions::ComplexRangeKind::CX_Full) + D.Diag(clang::diag::warn_drv_overriding_option) + << EnumComplexRangeToStr(Range1) << EnumComplexRangeToStr(Range2); +} + +static std::string RenderComplexRangeOption(std::string Range) { + std::string ComplexRangeStr = "-complex-range="; + ComplexRangeStr += Range; + return ComplexRangeStr; +} + static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, bool OFastEnabled, const ArgList &Args, ArgStringList &CmdArgs, @@ -2703,6 +2735,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, bool StrictFPModel = false; StringRef Float16ExcessPrecision = ""; StringRef BFloat16ExcessPrecision = ""; + LangOptions::ComplexRangeKind Range = LangOptions::ComplexRangeKind::CX_Full; if (const Arg *A = Args.getLastArg(options::OPT_flimited_precision_EQ)) { CmdArgs.push_back("-mlimit-float-precision"); @@ -2715,6 +2748,28 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, switch (optID) { default: break; + case options::OPT_fcx_limited_range: { + EmitComplexRangeDiag(D, Range, LangOptions::ComplexRangeKind::CX_Limited); + Range = LangOptions::ComplexRangeKind::CX_Limited; + std::string ComplexRangeStr = RenderComplexRangeOption("limited"); + if (!ComplexRangeStr.empty()) + CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr)); + break; + } + case options::OPT_fno_cx_limited_range: + Range = LangOptions::ComplexRangeKind::CX_Full; + break; + case options::OPT_fcx_fortran_rules: { + EmitComplexRangeDiag(D, Range, LangOptions::ComplexRangeKind::CX_Fortran); + Range = LangOptions::ComplexRangeKind::CX_Fortran; + std::string ComplexRangeStr = RenderComplexRangeOption("fortran"); + if (!ComplexRangeStr.empty()) + CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr)); + break; + } + case options::OPT_fno_cx_fortran_rules: + Range = LangOptions::ComplexRangeKind::CX_Full; + break; case options::OPT_ffp_model_EQ: { // If -ffp-model= is seen, reset to fno-fast-math HonorINFs = true; @@ -2769,7 +2824,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, D.Diag(diag::err_drv_unsupported_option_argument) << A->getSpelling() << Val; break; - } + } } switch (optID) { @@ -2968,7 +3023,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, if (!OFastEnabled) continue; [[fallthrough]]; - case options::OPT_ffast_math: + case options::OPT_ffast_math: { HonorINFs = false; HonorNaNs = false; MathErrno = false; @@ -2982,7 +3037,13 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, // If fast-math is set then set the fp-contract mode to fast. FPContract = "fast"; SeenUnsafeMathModeOption = true; + // ffast-math enables fortran rules for complex multiplication and + // division. + std::string ComplexRangeStr = RenderComplexRangeOption("limited"); + if (!ComplexRangeStr.empty()) + CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr)); break; + } case options::OPT_fno_fast_math: HonorINFs = true; HonorNaNs = true; @@ -3136,6 +3197,15 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, if (Args.hasFlag(options::OPT_fno_strict_float_cast_overflow, options::OPT_fstrict_float_cast_overflow, false)) CmdArgs.push_back("-fno-strict-float-cast-overflow"); + + if (const Arg *A = Args.getLastArg(options::OPT_fcx_limited_range)) + CmdArgs.push_back("-fcx-limited-range"); + if (const Arg *A = Args.getLastArg(options::OPT_fcx_fortran_rules)) + CmdArgs.push_back("-fcx-fortran-rules"); + if (const Arg *A = Args.getLastArg(options::OPT_fno_cx_limited_range)) + CmdArgs.push_back("-fno-cx-limited-range"); + if (const Arg *A = Args.getLastArg(options::OPT_fno_cx_fortran_rules)) + CmdArgs.push_back("-fno-cx-fortran-rules"); } static void RenderAnalyzerOptions(const ArgList &Args, ArgStringList &CmdArgs, diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 0ae8e2dce32e94a8703ea73cfae44f95f4a16a68..31e7d68161ff1b224dfa4b49433d82242b02794d 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1120,55 +1120,61 @@ void tools::addFortranRuntimeLibs(const ToolChain &TC, const ArgList &Args, llvm::opt::ArgStringList &CmdArgs) { // These are handled earlier on Windows by telling the frontend driver to add // the correct libraries to link against as dependents in the object file. - if (!TC.getTriple().isKnownWindowsMSVCEnvironment()) { - // The --whole-archive option needs to be part of the link line to - // make sure that the main() function from Fortran_main.a is pulled - // in by the linker. Determine if --whole-archive is active when - // flang will try to link Fortran_main.a. If it is, don't add the - // --whole-archive flag to the link line. If it's not, add a proper - // --whole-archive/--no-whole-archive bracket to the link line. - bool WholeArchiveActive = false; - for (auto *Arg : Args.filtered(options::OPT_Wl_COMMA)) - if (Arg) - for (StringRef ArgValue : Arg->getValues()) { - if (ArgValue == "--whole-archive") - WholeArchiveActive = true; - if (ArgValue == "--no-whole-archive") - WholeArchiveActive = false; - } - if (!WholeArchiveActive) - CmdArgs.push_back("--whole-archive"); - CmdArgs.push_back("-lFortran_main"); - if (!WholeArchiveActive) - CmdArgs.push_back("--no-whole-archive"); + // if -fno-fortran-main has been passed, skip linking Fortran_main.a + bool LinkFortranMain = !Args.hasArg(options::OPT_no_fortran_main); + if (!TC.getTriple().isKnownWindowsMSVCEnvironment()) { + if (LinkFortranMain) { + // The --whole-archive option needs to be part of the link line to + // make sure that the main() function from Fortran_main.a is pulled + // in by the linker. Determine if --whole-archive is active when + // flang will try to link Fortran_main.a. If it is, don't add the + // --whole-archive flag to the link line. If it's not, add a proper + // --whole-archive/--no-whole-archive bracket to the link line. + bool WholeArchiveActive = false; + for (auto *Arg : Args.filtered(options::OPT_Wl_COMMA)) + if (Arg) + for (StringRef ArgValue : Arg->getValues()) { + if (ArgValue == "--whole-archive") + WholeArchiveActive = true; + if (ArgValue == "--no-whole-archive") + WholeArchiveActive = false; + } + if (!WholeArchiveActive) + CmdArgs.push_back("--whole-archive"); + CmdArgs.push_back("-lFortran_main"); + if (!WholeArchiveActive) + CmdArgs.push_back("--no-whole-archive"); + } // Perform regular linkage of the remaining runtime libraries. CmdArgs.push_back("-lFortranRuntime"); CmdArgs.push_back("-lFortranDecimal"); } else { - unsigned RTOptionID = options::OPT__SLASH_MT; - if (auto *rtl = Args.getLastArg(options::OPT_fms_runtime_lib_EQ)) { - RTOptionID = llvm::StringSwitch(rtl->getValue()) - .Case("static", options::OPT__SLASH_MT) - .Case("static_dbg", options::OPT__SLASH_MTd) - .Case("dll", options::OPT__SLASH_MD) - .Case("dll_dbg", options::OPT__SLASH_MDd) - .Default(options::OPT__SLASH_MT); - } - switch (RTOptionID) { - case options::OPT__SLASH_MT: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static.lib"); - break; - case options::OPT__SLASH_MTd: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static_dbg.lib"); - break; - case options::OPT__SLASH_MD: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic.lib"); - break; - case options::OPT__SLASH_MDd: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic_dbg.lib"); - break; + if (LinkFortranMain) { + unsigned RTOptionID = options::OPT__SLASH_MT; + if (auto *rtl = Args.getLastArg(options::OPT_fms_runtime_lib_EQ)) { + RTOptionID = llvm::StringSwitch(rtl->getValue()) + .Case("static", options::OPT__SLASH_MT) + .Case("static_dbg", options::OPT__SLASH_MTd) + .Case("dll", options::OPT__SLASH_MD) + .Case("dll_dbg", options::OPT__SLASH_MDd) + .Default(options::OPT__SLASH_MT); + } + switch (RTOptionID) { + case options::OPT__SLASH_MT: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static.lib"); + break; + case options::OPT__SLASH_MTd: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static_dbg.lib"); + break; + case options::OPT__SLASH_MD: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic.lib"); + break; + case options::OPT__SLASH_MDd: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic_dbg.lib"); + break; + } } } } @@ -2583,9 +2589,7 @@ void tools::addMachineOutlinerArgs(const Driver &D, // We only support -moutline in AArch64 and ARM targets right now. If // we're not compiling for these, emit a warning and ignore the flag. // Otherwise, add the proper mllvm flags. - if (!(Triple.isARM() || Triple.isThumb() || - Triple.getArch() == llvm::Triple::aarch64 || - Triple.getArch() == llvm::Triple::aarch64_32)) { + if (!(Triple.isARM() || Triple.isThumb() || Triple.isAArch64())) { D.Diag(diag::warn_drv_moutline_unsupported_opt) << Triple.getArchName(); } else { addArg(Twine("-enable-machine-outliner")); diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp index e95ff98e6c940f1becedc9b40c74d8c249495a84..ef1e77974c1eaaf79f707bfc22c565024f3b5862 100644 --- a/clang/lib/Driver/ToolChains/Cuda.cpp +++ b/clang/lib/Driver/ToolChains/Cuda.cpp @@ -78,6 +78,10 @@ CudaVersion getCudaVersion(uint32_t raw_version) { return CudaVersion::CUDA_120; if (raw_version < 12020) return CudaVersion::CUDA_121; + if (raw_version < 12030) + return CudaVersion::CUDA_122; + if (raw_version < 12040) + return CudaVersion::CUDA_123; return CudaVersion::NEW; } @@ -671,6 +675,8 @@ void NVPTX::getNVPTXTargetFeatures(const Driver &D, const llvm::Triple &Triple, case CudaVersion::CUDA_##CUDA_VER: \ PtxFeature = "+ptx" #PTX_VER; \ break; + CASE_CUDA_VERSION(123, 83); + CASE_CUDA_VERSION(122, 82); CASE_CUDA_VERSION(121, 81); CASE_CUDA_VERSION(120, 80); CASE_CUDA_VERSION(118, 78); diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp index f09bc27d7d2c0e6d76aa776e98bb5b3d8ee15c1d..692b3a3f285d744afc868a622ea7dc0aa1559de6 100644 --- a/clang/lib/Driver/ToolChains/Darwin.cpp +++ b/clang/lib/Driver/ToolChains/Darwin.cpp @@ -1281,7 +1281,7 @@ void MachO::AddLinkRuntimeLib(const ArgList &Args, ArgStringList &CmdArgs, // rpaths. This is currently true from this place, but we need to be // careful if this function is ever called before user's rpaths are emitted. if (Opts & RLO_AddRPath) { - assert(DarwinLibName.endswith(".dylib") && "must be a dynamic library"); + assert(DarwinLibName.ends_with(".dylib") && "must be a dynamic library"); // Add @executable_path to rpath to support having the dylib copied with // the executable. diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp index 9b21fe952af7a8293bf92e41758a290563c3c2d1..502b9f17a06c52ff06434f206489f817f81dfcc4 100644 --- a/clang/lib/Driver/ToolChains/Flang.cpp +++ b/clang/lib/Driver/ToolChains/Flang.cpp @@ -231,6 +231,8 @@ static void processVSRuntimeLibrary(const ToolChain &TC, const ArgList &Args, ArgStringList &CmdArgs) { assert(TC.getTriple().isKnownWindowsMSVCEnvironment() && "can only add VS runtime library on Windows!"); + // if -fno-fortran-main has been passed, skip linking Fortran_main.a + bool LinkFortranMain = !Args.hasArg(options::OPT_no_fortran_main); if (TC.getTriple().isKnownWindowsMSVCEnvironment()) { CmdArgs.push_back(Args.MakeArgString( "--dependent-lib=" + TC.getCompilerRTBasename(Args, "builtins"))); @@ -248,7 +250,8 @@ static void processVSRuntimeLibrary(const ToolChain &TC, const ArgList &Args, case options::OPT__SLASH_MT: CmdArgs.push_back("-D_MT"); CmdArgs.push_back("--dependent-lib=libcmt"); - CmdArgs.push_back("--dependent-lib=Fortran_main.static.lib"); + if (LinkFortranMain) + CmdArgs.push_back("--dependent-lib=Fortran_main.static.lib"); CmdArgs.push_back("--dependent-lib=FortranRuntime.static.lib"); CmdArgs.push_back("--dependent-lib=FortranDecimal.static.lib"); break; @@ -256,7 +259,8 @@ static void processVSRuntimeLibrary(const ToolChain &TC, const ArgList &Args, CmdArgs.push_back("-D_MT"); CmdArgs.push_back("-D_DEBUG"); CmdArgs.push_back("--dependent-lib=libcmtd"); - CmdArgs.push_back("--dependent-lib=Fortran_main.static_dbg.lib"); + if (LinkFortranMain) + CmdArgs.push_back("--dependent-lib=Fortran_main.static_dbg.lib"); CmdArgs.push_back("--dependent-lib=FortranRuntime.static_dbg.lib"); CmdArgs.push_back("--dependent-lib=FortranDecimal.static_dbg.lib"); break; @@ -264,7 +268,8 @@ static void processVSRuntimeLibrary(const ToolChain &TC, const ArgList &Args, CmdArgs.push_back("-D_MT"); CmdArgs.push_back("-D_DLL"); CmdArgs.push_back("--dependent-lib=msvcrt"); - CmdArgs.push_back("--dependent-lib=Fortran_main.dynamic.lib"); + if (LinkFortranMain) + CmdArgs.push_back("--dependent-lib=Fortran_main.dynamic.lib"); CmdArgs.push_back("--dependent-lib=FortranRuntime.dynamic.lib"); CmdArgs.push_back("--dependent-lib=FortranDecimal.dynamic.lib"); break; @@ -273,7 +278,8 @@ static void processVSRuntimeLibrary(const ToolChain &TC, const ArgList &Args, CmdArgs.push_back("-D_DEBUG"); CmdArgs.push_back("-D_DLL"); CmdArgs.push_back("--dependent-lib=msvcrtd"); - CmdArgs.push_back("--dependent-lib=Fortran_main.dynamic_dbg.lib"); + if (LinkFortranMain) + CmdArgs.push_back("--dependent-lib=Fortran_main.dynamic_dbg.lib"); CmdArgs.push_back("--dependent-lib=FortranRuntime.dynamic_dbg.lib"); CmdArgs.push_back("--dependent-lib=FortranDecimal.dynamic_dbg.lib"); break; diff --git a/clang/lib/Driver/ToolChains/MSVC.cpp b/clang/lib/Driver/ToolChains/MSVC.cpp index 8a4a174c90ea8557336cc3258d617628828b6787..6d925555b7bb4b2d462ef91ab5eb0f0974ab1a0c 100644 --- a/clang/lib/Driver/ToolChains/MSVC.cpp +++ b/clang/lib/Driver/ToolChains/MSVC.cpp @@ -787,11 +787,11 @@ VersionTuple MSVCToolChain::computeMSVCVersion(const Driver *D, if (MSVT.empty() && Args.hasFlag(options::OPT_fms_extensions, options::OPT_fno_ms_extensions, IsWindowsMSVC)) { - // -fms-compatibility-version=19.20 is default, aka 2019, 16.x + // -fms-compatibility-version=19.33 is default, aka 2022, 17.3 // NOTE: when changing this value, also update // clang/docs/CommandGuide/clang.rst and clang/docs/UsersManual.rst // accordingly. - MSVT = VersionTuple(19, 20); + MSVT = VersionTuple(19, 33); } return MSVT; } diff --git a/clang/lib/Driver/ToolChains/WebAssembly.cpp b/clang/lib/Driver/ToolChains/WebAssembly.cpp index f04018179a5dab724aeb603568f4f2a4b283b021..f131b6cf3baff9d7ccfc2782d3ddfe6ea9b17b57 100644 --- a/clang/lib/Driver/ToolChains/WebAssembly.cpp +++ b/clang/lib/Driver/ToolChains/WebAssembly.cpp @@ -143,7 +143,7 @@ void wasm::Linker::ConstructJob(Compilation &C, const JobAction &JA, // When optimizing, if wasm-opt is available, run it. std::string WasmOptPath; - if (Arg *A = Args.getLastArg(options::OPT_O_Group)) { + if (Args.getLastArg(options::OPT_O_Group)) { WasmOptPath = ToolChain.GetProgramPath("wasm-opt"); if (WasmOptPath == "wasm-opt") { WasmOptPath = {}; diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp index 931933b2bd1ac0e98c2bebe3bd15f306971292df..d9675b0c94de3962fa185b87f9819bb2d844b268 100644 --- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp +++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp @@ -199,9 +199,10 @@ StringRef getLanguageName(Language Lang) { return "objective-c"; case Language::CXX: return "c++"; + case Language::ObjCXX: + return "objective-c++"; // Unsupported language currently - case Language::ObjCXX: case Language::OpenCL: case Language::OpenCLCXX: case Language::CUDA: diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp index cb7dc5f3dd9babcbfdbe15c3278331fbe1d0596f..9e4e939503dfe4a0b5dd501e14353cdb4133c664 100644 --- a/clang/lib/Format/ContinuationIndenter.cpp +++ b/clang/lib/Format/ContinuationIndenter.cpp @@ -1259,7 +1259,7 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) { } if (Style.AlignAfterOpenBracket == FormatStyle::BAS_BlockIndent && (Current.is(tok::r_paren) || - (Current.is(tok::r_brace) && + (Current.is(tok::r_brace) && Current.MatchingParen && Current.MatchingParen->is(BK_BracedInit))) && State.Stack.size() > 1) { return State.Stack[State.Stack.size() - 2].LastSpace; diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp index b09487435adb2da7a8f3908006b798d55d47dfc5..8feee7457fc31b76a09f9c9ebd9a9104db281404 100644 --- a/clang/lib/Format/Format.cpp +++ b/clang/lib/Format/Format.cpp @@ -3955,10 +3955,7 @@ llvm::Expected getStyle(StringRef StyleName, StringRef FileName, StringRef FallbackStyleName, StringRef Code, llvm::vfs::FileSystem *FS, bool AllowUnknownOptions) { - if (!FS) - FS = llvm::vfs::getRealFileSystem().get(); FormatStyle Style = getLLVMStyle(guessLanguage(FileName, Code)); - FormatStyle FallbackStyle = getNoStyle(); if (!getPredefinedStyle(FallbackStyleName, Style.Language, &FallbackStyle)) return make_string_error("Invalid fallback style: " + FallbackStyleName); @@ -3974,14 +3971,18 @@ llvm::Expected getStyle(StringRef StyleName, StringRef FileName, AllowUnknownOptions)) { return make_string_error("Error parsing -style: " + ec.message()); } - if (Style.InheritsParentConfig) { - ChildFormatTextToApply.emplace_back( - llvm::MemoryBuffer::getMemBuffer(StyleName, Source, false)); - } else { + + if (!Style.InheritsParentConfig) return Style; - } + + ChildFormatTextToApply.emplace_back( + llvm::MemoryBuffer::getMemBuffer(StyleName, Source, false)); } + if (!FS) + FS = llvm::vfs::getRealFileSystem().get(); + assert(FS); + // User provided clang-format file using -style=file:path/to/format/file. if (!Style.InheritsParentConfig && StyleName.starts_with_insensitive("file:")) { @@ -4015,18 +4016,12 @@ llvm::Expected getStyle(StringRef StyleName, StringRef FileName, return Style; } - // Reset possible inheritance - Style.InheritsParentConfig = false; - - // Look for .clang-format/_clang-format file in the file's parent directories. - SmallString<128> UnsuitableConfigFiles; SmallString<128> Path(FileName); if (std::error_code EC = FS->makeAbsolute(Path)) return make_string_error(EC.message()); - llvm::SmallVector FilesToLookFor; - FilesToLookFor.push_back(".clang-format"); - FilesToLookFor.push_back("_clang-format"); + // Reset possible inheritance + Style.InheritsParentConfig = false; auto dropDiagnosticHandler = [](const llvm::SMDiagnostic &, void *) {}; @@ -4040,9 +4035,14 @@ llvm::Expected getStyle(StringRef StyleName, StringRef FileName, } }; + // Look for .clang-format/_clang-format file in the file's parent directories. + llvm::SmallVector FilesToLookFor; + FilesToLookFor.push_back(".clang-format"); + FilesToLookFor.push_back("_clang-format"); + + SmallString<128> UnsuitableConfigFiles; for (StringRef Directory = Path; !Directory.empty(); Directory = llvm::sys::path::parent_path(Directory)) { - auto Status = FS->status(Directory); if (!Status || Status->getType() != llvm::sys::fs::file_type::directory_file) { @@ -4055,50 +4055,51 @@ llvm::Expected getStyle(StringRef StyleName, StringRef FileName, llvm::sys::path::append(ConfigFile, F); LLVM_DEBUG(llvm::dbgs() << "Trying " << ConfigFile << "...\n"); - Status = FS->status(ConfigFile.str()); - - if (Status && - (Status->getType() == llvm::sys::fs::file_type::regular_file)) { - llvm::ErrorOr> Text = - loadAndParseConfigFile(ConfigFile, FS, &Style, AllowUnknownOptions); - if (auto EC = Text.getError()) { - if (EC == ParseError::Unsuitable) { - if (!UnsuitableConfigFiles.empty()) - UnsuitableConfigFiles.append(", "); - UnsuitableConfigFiles.append(ConfigFile); - continue; - } + Status = FS->status(ConfigFile); + if (!Status || + Status->getType() != llvm::sys::fs::file_type::regular_file) { + continue; + } + + llvm::ErrorOr> Text = + loadAndParseConfigFile(ConfigFile, FS, &Style, AllowUnknownOptions); + if (auto EC = Text.getError()) { + if (EC != ParseError::Unsuitable) { return make_string_error("Error reading " + ConfigFile + ": " + EC.message()); } - LLVM_DEBUG(llvm::dbgs() - << "Using configuration file " << ConfigFile << "\n"); + if (!UnsuitableConfigFiles.empty()) + UnsuitableConfigFiles.append(", "); + UnsuitableConfigFiles.append(ConfigFile); + continue; + } - if (!Style.InheritsParentConfig) { - if (ChildFormatTextToApply.empty()) - return Style; + LLVM_DEBUG(llvm::dbgs() + << "Using configuration file " << ConfigFile << "\n"); + if (!Style.InheritsParentConfig) { + if (!ChildFormatTextToApply.empty()) { LLVM_DEBUG(llvm::dbgs() << "Applying child configurations\n"); applyChildFormatTexts(&Style); - - return Style; } + return Style; + } - LLVM_DEBUG(llvm::dbgs() << "Inherits parent configuration\n"); + LLVM_DEBUG(llvm::dbgs() << "Inherits parent configuration\n"); - // Reset inheritance of style - Style.InheritsParentConfig = false; + // Reset inheritance of style + Style.InheritsParentConfig = false; - ChildFormatTextToApply.emplace_back(std::move(*Text)); + ChildFormatTextToApply.emplace_back(std::move(*Text)); - // Breaking out of the inner loop, since we don't want to parse - // .clang-format AND _clang-format, if both exist. Then we continue the - // inner loop (parent directories) in search for the parent - // configuration. - break; - } + // Breaking out of the inner loop, since we don't want to parse + // .clang-format AND _clang-format, if both exist. Then we continue the + // outer loop (parent directories) in search for the parent + // configuration. + break; } } + if (!UnsuitableConfigFiles.empty()) { return make_string_error("Configuration file(s) do(es) not support " + getLanguageName(Style.Language) + ": " + diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp index b4930c2e4621dc200f1901f748de3cda3304201d..56077499c39d539ab69a94786a202d89af3e264c 100644 --- a/clang/lib/Format/UnwrappedLineFormatter.cpp +++ b/clang/lib/Format/UnwrappedLineFormatter.cpp @@ -411,9 +411,16 @@ private: } } + const auto *LastNonComment = TheLine->getLastNonComment(); + assert(LastNonComment); + // FIXME: There are probably cases where we should use LastNonComment + // instead of TheLine->Last. + // Try to merge a function block with left brace unwrapped. - if (TheLine->Last->is(TT_FunctionLBrace) && TheLine->First != TheLine->Last) + if (LastNonComment->is(TT_FunctionLBrace) && + TheLine->First != LastNonComment) { return MergeShortFunctions ? tryMergeSimpleBlock(I, E, Limit) : 0; + } // Try to merge a control statement block with left brace unwrapped. if (TheLine->Last->is(tok::l_brace) && FirstNonComment != TheLine->Last && FirstNonComment->isOneOf(tok::kw_if, tok::kw_while, tok::kw_for, @@ -789,7 +796,8 @@ private: } } - if (Line.Last->is(tok::l_brace)) { + if (const auto *LastNonComment = Line.getLastNonComment(); + LastNonComment && LastNonComment->is(tok::l_brace)) { if (IsSplitBlock && Line.First == Line.Last && I > AnnotatedLines.begin() && (I[-1]->endsWith(tok::kw_else) || IsCtrlStmt(*I[-1]))) { @@ -805,7 +813,8 @@ private: if (ShouldMerge()) { // We merge empty blocks even if the line exceeds the column limit. - Tok->SpacesRequiredBefore = Style.SpaceInEmptyBlock ? 1 : 0; + Tok->SpacesRequiredBefore = + (Style.SpaceInEmptyBlock || Line.Last->is(tok::comment)) ? 1 : 0; Tok->CanBreakBefore = true; return 1; } else if (Limit != 0 && !Line.startsWithNamespace() && diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index e5f8c0746a99dd457bd4727d83f82531474fd70c..56bbef9697b650073778b1db61fdbb2c49ec8b38 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -2260,7 +2260,7 @@ GlobalModuleIndex *CompilerInstance::loadGlobalModuleIndex( for (ModuleMap::module_iterator I = MMap.module_begin(), E = MMap.module_end(); I != E; ++I) { Module *TheModule = I->second; - const FileEntry *Entry = TheModule->getASTFile(); + OptionalFileEntryRef Entry = TheModule->getASTFile(); if (!Entry) { SmallVector, 2> Path; Path.push_back(std::make_pair( diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 56de0f75928ca4245aaa54e900e1636659099274..b33bdad2ad81bad041c9ca791066ac347706d1e7 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -4365,6 +4365,9 @@ static void GeneratePreprocessorArgs(const PreprocessorOptions &Opts, if (Opts.SourceDateEpoch) GenerateArg(Consumer, OPT_source_date_epoch, Twine(*Opts.SourceDateEpoch)); + if (Opts.DefineTargetOSMacros) + GenerateArg(Consumer, OPT_fdefine_target_os_macros); + // Don't handle LexEditorPlaceholders. It is implied by the action that is // generated elsewhere. } @@ -4463,6 +4466,10 @@ static bool ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args, if (isStrictlyPreprocessorAction(Action)) Opts.LexEditorPlaceholders = false; + Opts.DefineTargetOSMacros = + Args.hasFlag(OPT_fdefine_target_os_macros, + OPT_fno_define_target_os_macros, Opts.DefineTargetOSMacros); + return Diags.getNumErrors() == NumErrorsBefore; } diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index 2afcf1cf9f68c819d6ddc59da5bbcc5c70063827..c1d6e71455365c980b7e8cef3160698c50e40595 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -258,6 +258,16 @@ bool GenerateModuleInterfaceAction::BeginSourceFileAction( return GenerateModuleAction::BeginSourceFileAction(CI); } +std::unique_ptr +GenerateModuleInterfaceAction::CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) { + CI.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true; + CI.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true; + CI.getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings = true; + + return GenerateModuleAction::CreateASTConsumer(CI, InFile); +} + std::unique_ptr GenerateModuleInterfaceAction::CreateOutputFile(CompilerInstance &CI, StringRef InFile) { diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp index 17948dcebd7e55a2d7412889ef1f20c91717e135..d83128adb511ef4f999cfebf529bf6ac998d74c6 100644 --- a/clang/lib/Frontend/InitPreprocessor.cpp +++ b/clang/lib/Frontend/InitPreprocessor.cpp @@ -809,6 +809,13 @@ static void InitializePredefinedMacros(const TargetInfo &TI, Builder.defineMacro("__ATOMIC_ACQ_REL", "4"); Builder.defineMacro("__ATOMIC_SEQ_CST", "5"); + // Define macros for the clang atomic scopes. + Builder.defineMacro("__MEMORY_SCOPE_SYSTEM", "0"); + Builder.defineMacro("__MEMORY_SCOPE_DEVICE", "1"); + Builder.defineMacro("__MEMORY_SCOPE_WRKGRP", "2"); + Builder.defineMacro("__MEMORY_SCOPE_WVFRNT", "3"); + Builder.defineMacro("__MEMORY_SCOPE_SINGLE", "4"); + // Define macros for the OpenCL memory scope. // The values should match AtomicScopeOpenCLModel::ID enum. static_assert( @@ -1344,6 +1351,15 @@ static void InitializePredefinedMacros(const TargetInfo &TI, if (TI.getTriple().isOSBinFormatELF()) Builder.defineMacro("__ELF__"); + // Target OS macro definitions. + if (PPOpts.DefineTargetOSMacros) { + const llvm::Triple &Triple = TI.getTriple(); +#define TARGET_OS(Name, Predicate) \ + Builder.defineMacro(#Name, (Predicate) ? "1" : "0"); +#include "clang/Basic/TargetOSMacros.def" +#undef TARGET_OS + } + // Get other target #defines. TI.getTargetDefines(LangOpts, Builder); } diff --git a/clang/lib/Lex/HeaderSearch.cpp b/clang/lib/Lex/HeaderSearch.cpp index cf1c0cc5284f316db41324f8c2c934ec98c99029..c03cf19688d961adb9a02e33b4390b6cbf2b4748 100644 --- a/clang/lib/Lex/HeaderSearch.cpp +++ b/clang/lib/Lex/HeaderSearch.cpp @@ -105,7 +105,7 @@ void HeaderSearch::PrintStats() { void HeaderSearch::SetSearchPaths( std::vector dirs, unsigned int angledDirIdx, - unsigned int systemDirIdx, bool noCurDirSearch, + unsigned int systemDirIdx, llvm::DenseMap searchDirToHSEntry) { assert(angledDirIdx <= systemDirIdx && systemDirIdx <= dirs.size() && "Directory indices are unordered"); @@ -113,7 +113,6 @@ void HeaderSearch::SetSearchPaths( SearchDirsUsage.assign(SearchDirs.size(), false); AngledDirIdx = angledDirIdx; SystemDirIdx = systemDirIdx; - NoCurDirSearch = noCurDirSearch; SearchDirToHSEntry = std::move(searchDirToHSEntry); //LookupFileCache.clear(); indexInitialHeaderMaps(); @@ -904,12 +903,12 @@ OptionalFileEntryRef HeaderSearch::LookupFile( ModuleMap::KnownHeader MSSuggestedModule; OptionalFileEntryRef MSFE; - // Unless disabled, check to see if the file is in the #includer's - // directory. This cannot be based on CurDir, because each includer could be - // a #include of a subdirectory (#include "foo/bar.h") and a subsequent - // include of "baz.h" should resolve to "whatever/foo/baz.h". - // This search is not done for <> headers. - if (!Includers.empty() && !isAngled && !NoCurDirSearch) { + // Check to see if the file is in the #includer's directory. This cannot be + // based on CurDir, because each includer could be a #include of a + // subdirectory (#include "foo/bar.h") and a subsequent include of "baz.h" + // should resolve to "whatever/foo/baz.h". This search is not done for <> + // headers. + if (!Includers.empty() && !isAngled) { SmallString<1024> TmpDir; bool First = true; for (const auto &IncluderAndDir : Includers) { diff --git a/clang/lib/Lex/InitHeaderSearch.cpp b/clang/lib/Lex/InitHeaderSearch.cpp index 1f4c7a6d67b7f79bc2bb7ee22d2bcd4de80c99e0..5b1b7c859c85d9346858ee7472f6cfd3d9838f8e 100644 --- a/clang/lib/Lex/InitHeaderSearch.cpp +++ b/clang/lib/Lex/InitHeaderSearch.cpp @@ -513,9 +513,8 @@ void InitHeaderSearch::Realize(const LangOptions &Lang) { unsigned NonSystemRemoved = RemoveDuplicates(SearchList, NumQuoted, Verbose); NumAngled -= NonSystemRemoved; - bool DontSearchCurDir = false; // TODO: set to true if -I- is set? Headers.SetSearchPaths(extractLookups(SearchList), NumQuoted, NumAngled, - DontSearchCurDir, mapToUserEntries(SearchList)); + mapToUserEntries(SearchList)); Headers.SetSystemHeaderPrefixes(SystemHeaderPrefixes); diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp index 1d67e275cb4775a15dce3a9543a142d88d914573..d35c282543c564d6ac94f8db51d50d38f5a4f7db 100644 --- a/clang/lib/Lex/ModuleMap.cpp +++ b/clang/lib/Lex/ModuleMap.cpp @@ -1067,9 +1067,7 @@ Module *ModuleMap::inferFrameworkModule(DirectoryEntryRef FrameworkDir, if (!canInfer) return nullptr; } else { - OptionalFileEntryRefDegradesToFileEntryPtr ModuleMapRef = - getModuleMapFileForUniquing(Parent); - ModuleMapFile = ModuleMapRef; + ModuleMapFile = getModuleMapFileForUniquing(Parent); } // Look for an umbrella header. @@ -1866,7 +1864,7 @@ void ModuleMapParser::diagnosePrivateModules(SourceLocation ExplicitLoc, continue; SmallString<128> FullName(ActiveModule->getFullModuleName()); - if (!FullName.startswith(M->Name) && !FullName.endswith("Private")) + if (!FullName.starts_with(M->Name) && !FullName.ends_with("Private")) continue; SmallString<128> FixedPrivModDecl; SmallString<128> Canonical(M->Name); diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index 956e2276f25b710174fdb1c34b51c0948cc0a9a0..14003480d7fa2e91f38a363a7932879fec2f2d9e 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -1934,7 +1934,8 @@ Preprocessor::getIncludeNextStart(const Token &IncludeNextTok) const { // Start looking up in the directory *after* the one in which the current // file would be found, if any. assert(CurPPLexer && "#include_next directive in macro?"); - LookupFromFile = CurPPLexer->getFileEntry(); + if (auto FE = CurPPLexer->getFileEntry()) + LookupFromFile = *FE; Lookup = nullptr; } else if (!Lookup) { // The current file was not found by walking the include path. Either it diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp index 35ab42cb6b5ef852e3a1c34dba0a5115fbcd64b8..499813f8ab7df0e6f81f8b84e2d8ea5b6cd5399b 100644 --- a/clang/lib/Lex/Pragma.cpp +++ b/clang/lib/Lex/Pragma.cpp @@ -548,7 +548,7 @@ void Preprocessor::HandlePragmaDependency(Token &DependencyTok) { return; } - const FileEntry *CurFile = getCurrentFileLexer()->getFileEntry(); + OptionalFileEntryRef CurFile = getCurrentFileLexer()->getFileEntry(); // If this file is older than the file it depends on, emit a diagnostic. if (CurFile && CurFile->getModificationTime() < File->getModificationTime()) { diff --git a/clang/lib/Lex/PreprocessorLexer.cpp b/clang/lib/Lex/PreprocessorLexer.cpp index 23c80d375214c69b6b7343b96888835e811fe8b4..7551ba235fe9b85b331c913c752e2d4dacb418b8 100644 --- a/clang/lib/Lex/PreprocessorLexer.cpp +++ b/clang/lib/Lex/PreprocessorLexer.cpp @@ -47,7 +47,6 @@ void PreprocessorLexer::LexIncludeFilename(Token &FilenameTok) { /// getFileEntry - Return the FileEntry corresponding to this FileID. Like /// getFileID(), this only works for lexers with attached preprocessors. -OptionalFileEntryRefDegradesToFileEntryPtr -PreprocessorLexer::getFileEntry() const { +OptionalFileEntryRef PreprocessorLexer::getFileEntry() const { return PP->getSourceManager().getFileEntryRefForID(getFileID()); } diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp index 71cb665a563271ca77c517aa6e96c6c8758f9dbd..f7f096762e91a64b25eca104d1c352bbff9f4480 100644 --- a/clang/lib/Parse/ParseOpenACC.cpp +++ b/clang/lib/Parse/ParseOpenACC.cpp @@ -56,6 +56,7 @@ OpenACCDirectiveKindEx getOpenACCDirectiveKind(Token Tok) { .Case("shutdown", OpenACCDirectiveKind::Shutdown) .Case("set", OpenACCDirectiveKind::Shutdown) .Case("update", OpenACCDirectiveKind::Update) + .Case("wait", OpenACCDirectiveKind::Wait) .Default(OpenACCDirectiveKind::Invalid); if (DirKind != OpenACCDirectiveKind::Invalid) @@ -82,6 +83,27 @@ OpenACCAtomicKind getOpenACCAtomicKind(Token Tok) { .Default(OpenACCAtomicKind::Invalid); } +enum class OpenACCSpecialTokenKind { + ReadOnly, + DevNum, + Queues, +}; + +bool isOpenACCSpecialToken(OpenACCSpecialTokenKind Kind, Token Tok) { + if (!Tok.is(tok::identifier)) + return false; + + switch (Kind) { + case OpenACCSpecialTokenKind::ReadOnly: + return Tok.getIdentifierInfo()->isStr("readonly"); + case OpenACCSpecialTokenKind::DevNum: + return Tok.getIdentifierInfo()->isStr("devnum"); + case OpenACCSpecialTokenKind::Queues: + return Tok.getIdentifierInfo()->isStr("queues"); + } + llvm_unreachable("Unknown 'Kind' Passed"); +} + bool isOpenACCDirectiveKind(OpenACCDirectiveKind Kind, Token Tok) { if (!Tok.is(tok::identifier)) return false; @@ -123,6 +145,8 @@ bool isOpenACCDirectiveKind(OpenACCDirectiveKind Kind, Token Tok) { return Tok.getIdentifierInfo()->isStr("set"); case OpenACCDirectiveKind::Update: return Tok.getIdentifierInfo()->isStr("update"); + case OpenACCDirectiveKind::Wait: + return Tok.getIdentifierInfo()->isStr("wait"); case OpenACCDirectiveKind::Invalid: return false; } @@ -182,7 +206,7 @@ OpenACCDirectiveKind ParseOpenACCDirectiveKind(Parser &P) { // Just #pragma acc can get us immediately to the end, make sure we don't // introspect on the spelling before then. - if (FirstTok.isAnnotation()) { + if (FirstTok.isNot(tok::identifier)) { P.Diag(FirstTok, diag::err_acc_missing_directive); return OpenACCDirectiveKind::Invalid; } @@ -200,11 +224,8 @@ OpenACCDirectiveKind ParseOpenACCDirectiveKind(Parser &P) { if (ExDirKind >= OpenACCDirectiveKindEx::Invalid) { switch (ExDirKind) { case OpenACCDirectiveKindEx::Invalid: { - if (!FirstTok.is(tok::identifier)) - P.Diag(FirstTok, diag::err_expected) << tok::identifier; - else - P.Diag(FirstTok, diag::err_acc_invalid_directive) - << 0 << FirstTok.getIdentifierInfo(); + P.Diag(FirstTok, diag::err_acc_invalid_directive) + << 0 << FirstTok.getIdentifierInfo(); return OpenACCDirectiveKind::Invalid; } case OpenACCDirectiveKindEx::Enter: @@ -251,6 +272,67 @@ void ParseOpenACCClauseList(Parser &P) { } // namespace +/// OpenACC 3.3, section 2.16: +/// In this section and throughout the specification, the term wait-argument +/// means: +/// [ devnum : int-expr : ] [ queues : ] async-argument-list +bool Parser::ParseOpenACCWaitArgument() { + // [devnum : int-expr : ] + if (isOpenACCSpecialToken(OpenACCSpecialTokenKind::DevNum, Tok) && + NextToken().is(tok::colon)) { + // Consume devnum. + ConsumeToken(); + // Consume colon. + ConsumeToken(); + + ExprResult IntExpr = + getActions().CorrectDelayedTyposInExpr(ParseAssignmentExpression()); + if (IntExpr.isInvalid()) + return true; + + if (ExpectAndConsume(tok::colon)) + return true; + } + + // [ queues : ] + if (isOpenACCSpecialToken(OpenACCSpecialTokenKind::Queues, Tok) && + NextToken().is(tok::colon)) { + // Consume queues. + ConsumeToken(); + // Consume colon. + ConsumeToken(); + } + + // OpenACC 3.3, section 2.16: + // the term 'async-argument' means a nonnegative scalar integer expression, or + // one of the special values 'acc_async_noval' or 'acc_async_sync', as defined + // in the C header file and the Fortran opacc module. + // + // We are parsing this simply as list of assignment expressions (to avoid + // comma being troublesome), and will ensure it is an integral type. The + // 'special' types are defined as macros, so we can't really check those + // (other than perhaps as values at one point?), but the standard does say it + // is implementation-defined to use any other negative value. + // + // + bool FirstArg = true; + while (!getCurToken().isOneOf(tok::r_paren, tok::annot_pragma_openacc_end)) { + if (!FirstArg) { + if (ExpectAndConsume(tok::comma)) + return true; + } + FirstArg = false; + + ExprResult CurArg = + getActions().CorrectDelayedTyposInExpr(ParseAssignmentExpression()); + + if (CurArg.isInvalid()) + return true; + } + + return false; +} + ExprResult Parser::ParseOpenACCIDExpression() { ExprResult Res; if (getLangOpts().CPlusPlus) { @@ -340,8 +422,7 @@ void Parser::ParseOpenACCCacheVarList() { // specifications. First, see if we have `readonly:`, else we back-out and // treat it like the beginning of a reference to a potentially-existing // `readonly` variable. - if (getCurToken().is(tok::identifier) && - getCurToken().getIdentifierInfo()->isStr("readonly") && + if (isOpenACCSpecialToken(OpenACCSpecialTokenKind::ReadOnly, Tok) && NextToken().is(tok::colon)) { // Consume both tokens. ConsumeToken(); @@ -399,6 +480,13 @@ void Parser::ParseOpenACCDirective() { // so we can always consume the close. T.consumeClose(); break; + case OpenACCDirectiveKind::Wait: + // OpenACC has an optional paren-wrapped 'wait-argument'. + if (ParseOpenACCWaitArgument()) + T.skipToEnd(); + else + T.consumeClose(); + break; } } else if (DirKind == OpenACCDirectiveKind::Cache) { // Cache's paren var-list is required, so error here if it isn't provided. diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp index efdf7c90f977fbff32c7edae42e87dd04e3ef225..730ac1a0fee5cc9f151759cac31706869983760c 100644 --- a/clang/lib/Parse/ParsePragma.cpp +++ b/clang/lib/Parse/ParsePragma.cpp @@ -137,7 +137,20 @@ struct PragmaSTDC_CX_LIMITED_RANGEHandler : public PragmaHandler { void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer, Token &Tok) override { tok::OnOffSwitch OOS; - PP.LexOnOffSwitch(OOS); + if (PP.LexOnOffSwitch(OOS)) + return; + + MutableArrayRef Toks( + PP.getPreprocessorAllocator().Allocate(1), 1); + + Toks[0].startToken(); + Toks[0].setKind(tok::annot_pragma_cx_limited_range); + Toks[0].setLocation(Tok.getLocation()); + Toks[0].setAnnotationEndLoc(Tok.getLocation()); + Toks[0].setAnnotationValue( + reinterpret_cast(static_cast(OOS))); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, + /*IsReinject=*/false); } }; @@ -888,6 +901,31 @@ void Parser::HandlePragmaFEnvRound() { Actions.ActOnPragmaFEnvRound(PragmaLoc, RM); } +void Parser::HandlePragmaCXLimitedRange() { + assert(Tok.is(tok::annot_pragma_cx_limited_range)); + tok::OnOffSwitch OOS = static_cast( + reinterpret_cast(Tok.getAnnotationValue())); + + LangOptions::ComplexRangeKind Range; + switch (OOS) { + case tok::OOS_ON: + Range = LangOptions::CX_Limited; + break; + case tok::OOS_OFF: + Range = LangOptions::CX_Full; + break; + case tok::OOS_DEFAULT: + // According to ISO C99 standard chapter 7.3.4, the default value + // for the pragma is ``off'. -fcx-limited-range and -fcx-fortran-rules + // control the default value of these pragmas. + Range = getLangOpts().getComplexRange(); + break; + } + + SourceLocation PragmaLoc = ConsumeAnnotationToken(); + Actions.ActOnPragmaCXLimitedRange(PragmaLoc, Range); +} + StmtResult Parser::HandlePragmaCaptured() { assert(Tok.is(tok::annot_pragma_captured)); diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp index 918afdc2baea389e3d7972c6baa40f160d4168a3..d0ff33bd1379ab727bdf712ad2ebee58f64f9149 100644 --- a/clang/lib/Parse/ParseStmt.cpp +++ b/clang/lib/Parse/ParseStmt.cpp @@ -444,6 +444,14 @@ Retry: ConsumeAnnotationToken(); return StmtError(); + case tok::annot_pragma_cx_limited_range: + ProhibitAttributes(CXX11Attrs); + ProhibitAttributes(GNUAttrs); + Diag(Tok, diag::err_pragma_file_or_compound_scope) + << "STDC CX_LIMITED_RANGE"; + ConsumeAnnotationToken(); + return StmtError(); + case tok::annot_pragma_float_control: ProhibitAttributes(CXX11Attrs); ProhibitAttributes(GNUAttrs); @@ -1066,6 +1074,9 @@ void Parser::ParseCompoundStatementLeadingPragmas() { case tok::annot_pragma_fenv_round: HandlePragmaFEnvRound(); break; + case tok::annot_pragma_cx_limited_range: + HandlePragmaCXLimitedRange(); + break; case tok::annot_pragma_float_control: HandlePragmaFloatControl(); break; diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp index 1baeb2aeb021faa89826ef88616568c40ac11678..ec67faf7dcaf863985a2b595e517d3007ff3fa68 100644 --- a/clang/lib/Parse/Parser.cpp +++ b/clang/lib/Parse/Parser.cpp @@ -844,6 +844,9 @@ Parser::ParseExternalDeclaration(ParsedAttributes &Attrs, case tok::annot_pragma_fenv_round: HandlePragmaFEnvRound(); return nullptr; + case tok::annot_pragma_cx_limited_range: + HandlePragmaCXLimitedRange(); + return nullptr; case tok::annot_pragma_float_control: HandlePragmaFloatControl(); return nullptr; diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp index 8de144486c91df6297c0ecee79bb9af0a66339e1..1a1febf7a3524118d9179d14575e3c870a5a20c3 100644 --- a/clang/lib/Sema/HLSLExternalSemaSource.cpp +++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp @@ -116,11 +116,11 @@ struct BuiltinTypeDeclBuilder { } BuiltinTypeDeclBuilder &annotateResourceClass(ResourceClass RC, - ResourceKind RK) { + ResourceKind RK, bool IsROV) { if (Record->isCompleteDefinition()) return *this; - Record->addAttr( - HLSLResourceAttr::CreateImplicit(Record->getASTContext(), RC, RK)); + Record->addAttr(HLSLResourceAttr::CreateImplicit(Record->getASTContext(), + RC, RK, IsROV)); return *this; } @@ -306,6 +306,7 @@ struct BuiltinTypeDeclBuilder { } TemplateParameterListBuilder addTemplateArgumentList(); + BuiltinTypeDeclBuilder &addSimpleTemplateParams(ArrayRef Names); }; struct TemplateParameterListBuilder { @@ -360,11 +361,19 @@ struct TemplateParameterListBuilder { return Builder; } }; +} // namespace TemplateParameterListBuilder BuiltinTypeDeclBuilder::addTemplateArgumentList() { return TemplateParameterListBuilder(*this); } -} // namespace + +BuiltinTypeDeclBuilder & +BuiltinTypeDeclBuilder::addSimpleTemplateParams(ArrayRef Names) { + TemplateParameterListBuilder Builder = this->addTemplateArgumentList(); + for (StringRef Name : Names) + Builder.addTypeParameter(Name); + return Builder.finalizeTemplateArgs(); +} HLSLExternalSemaSource::~HLSLExternalSemaSource() {} @@ -390,7 +399,7 @@ void HLSLExternalSemaSource::InitializeSema(Sema &S) { // Force external decls in the HLSL namespace to load from the PCH. (void)HLSLNamespace->getCanonicalDecl()->decls_begin(); defineTrivialHLSLTypes(); - forwardDeclareHLSLTypes(); + defineHLSLTypesWithForwardDeclarations(); // This adds a `using namespace hlsl` directive. In DXC, we don't put HLSL's // built in types inside a namespace, but we are planning to change that in @@ -467,18 +476,43 @@ void HLSLExternalSemaSource::defineTrivialHLSLTypes() { .Record; } -void HLSLExternalSemaSource::forwardDeclareHLSLTypes() { +/// Set up common members and attributes for buffer types +static BuiltinTypeDeclBuilder setupBufferType(CXXRecordDecl *Decl, Sema &S, + ResourceClass RC, ResourceKind RK, + bool IsROV) { + return BuiltinTypeDeclBuilder(Decl) + .addHandleMember() + .addDefaultHandleConstructor(S, RC) + .annotateResourceClass(RC, RK, IsROV); +} + +void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() { CXXRecordDecl *Decl; Decl = BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace, "RWBuffer") - .addTemplateArgumentList() - .addTypeParameter("element_type", SemaPtr->getASTContext().FloatTy) - .finalizeTemplateArgs() + .addSimpleTemplateParams({"element_type"}) .Record; - if (!Decl->isCompleteDefinition()) - Completions.insert( - std::make_pair(Decl->getCanonicalDecl(), - std::bind(&HLSLExternalSemaSource::completeBufferType, - this, std::placeholders::_1))); + onCompletion(Decl, [this](CXXRecordDecl *Decl) { + setupBufferType(Decl, *SemaPtr, ResourceClass::UAV, + ResourceKind::TypedBuffer, /*IsROV=*/false) + .addArraySubscriptOperators() + .completeDefinition(); + }); + + Decl = + BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace, "RasterizerOrderedBuffer") + .addSimpleTemplateParams({"element_type"}) + .Record; + onCompletion(Decl, [this](CXXRecordDecl *Decl) { + setupBufferType(Decl, *SemaPtr, ResourceClass::UAV, + ResourceKind::TypedBuffer, /*IsROV=*/true) + .addArraySubscriptOperators() + .completeDefinition(); + }); +} + +void HLSLExternalSemaSource::onCompletion(CXXRecordDecl *Record, + CompletionFunction Fn) { + Completions.insert(std::make_pair(Record->getCanonicalDecl(), Fn)); } void HLSLExternalSemaSource::CompleteType(TagDecl *Tag) { @@ -496,12 +530,3 @@ void HLSLExternalSemaSource::CompleteType(TagDecl *Tag) { return; It->second(Record); } - -void HLSLExternalSemaSource::completeBufferType(CXXRecordDecl *Record) { - BuiltinTypeDeclBuilder(Record) - .addHandleMember() - .addDefaultHandleConstructor(*SemaPtr, ResourceClass::UAV) - .addArraySubscriptOperators() - .annotateResourceClass(ResourceClass::UAV, ResourceKind::TypedBuffer) - .completeDefinition(); -} diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp index 2c7ecf4610de0f064127eface82bb132377c034c..22929aa6316d7fc725e3f77396af528869c9f14f 100644 --- a/clang/lib/Sema/Sema.cpp +++ b/clang/lib/Sema/Sema.cpp @@ -2077,7 +2077,7 @@ void Sema::checkTypeSupport(QualType Ty, SourceLocation Loc, ValueDecl *D) { targetDiag(D->getLocation(), diag::note_defined_here, FD) << D; } - if (TI.hasRISCVVTypes() && Ty->isRVVType()) + if (TI.hasRISCVVTypes() && Ty->isRVVSizelessBuiltinType()) checkRVVTypeSupport(Ty, Loc, D); // Don't allow SVE types in functions without a SVE target. diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp index 79271c872627310ec92d1cda432f45897c07cb4b..0dcf42e489971344d516153c872a36c1df03e188 100644 --- a/clang/lib/Sema/SemaAttr.cpp +++ b/clang/lib/Sema/SemaAttr.cpp @@ -1352,6 +1352,14 @@ void Sema::ActOnPragmaFEnvAccess(SourceLocation Loc, bool IsEnabled) { CurFPFeatures = NewFPFeatures.applyOverrides(getLangOpts()); } +void Sema::ActOnPragmaCXLimitedRange(SourceLocation Loc, + LangOptions::ComplexRangeKind Range) { + FPOptionsOverride NewFPFeatures = CurFPFeatureOverrides(); + NewFPFeatures.setComplexRangeOverride(Range); + FpPragmaStack.Act(Loc, PSK_Set, StringRef(), NewFPFeatures); + CurFPFeatures = NewFPFeatures.applyOverrides(getLangOpts()); +} + void Sema::ActOnPragmaFPExceptions(SourceLocation Loc, LangOptions::FPExceptionModeKind FPE) { setExceptionMode(Loc, FPE); diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index fc6ee6b2c5ab4fd9754df06d5761c47f111cf350..cdb6e9584e9554104e3b4921f15d4752440661fe 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -5082,12 +5082,10 @@ static bool CheckInvalidVLENandLMUL(const TargetInfo &TI, CallExpr *TheCall, assert((EGW == 128 || EGW == 256) && "EGW can only be 128 or 256 bits"); // LMUL * VLEN >= EGW - unsigned ElemSize = Type->isRVVType(32, false) ? 32 : 64; - unsigned MinElemCount = Type->isRVVType(1) ? 1 - : Type->isRVVType(2) ? 2 - : Type->isRVVType(4) ? 4 - : Type->isRVVType(8) ? 8 - : 16; + ASTContext::BuiltinVectorTypeInfo Info = + S.Context.getBuiltinVectorTypeInfo(Type->castAs()); + unsigned ElemSize = S.Context.getTypeSize(Info.ElementType); + unsigned MinElemCount = Info.EC.getKnownMinValue(); unsigned EGS = EGW / ElemSize; // If EGS is less than or equal to the minimum number of elements, then the @@ -5215,15 +5213,13 @@ bool Sema::CheckRISCVBuiltinFunctionCall(const TargetInfo &TI, case RISCVVector::BI__builtin_rvv_vsmul_vx_tum: case RISCVVector::BI__builtin_rvv_vsmul_vv_tumu: case RISCVVector::BI__builtin_rvv_vsmul_vx_tumu: { - bool RequireV = false; - for (unsigned ArgNum = 0; ArgNum < TheCall->getNumArgs(); ++ArgNum) - RequireV |= TheCall->getArg(ArgNum)->getType()->isRVVType( - /* Bitwidth */ 64, /* IsFloat */ false); + ASTContext::BuiltinVectorTypeInfo Info = Context.getBuiltinVectorTypeInfo( + TheCall->getType()->castAs()); - if (RequireV && !TI.hasFeature("v")) + if (Context.getTypeSize(Info.ElementType) == 64 && !TI.hasFeature("v")) return Diag(TheCall->getBeginLoc(), diag::err_riscv_builtin_requires_extension) - << /* IsExtension */ false << TheCall->getSourceRange() << "v"; + << /* IsExtension */ true << TheCall->getSourceRange() << "v"; break; } @@ -5983,7 +5979,7 @@ bool Sema::CheckRISCVBuiltinFunctionCall(const TargetInfo &TI, ValType = ValType.getUnqualifiedType(); if (!ValType->isIntegerType() && !ValType->isAnyPointerType() && !ValType->isBlockPointerType() && !ValType->isFloatingType() && - !ValType->isVectorType() && !ValType->isRVVType()) { + !ValType->isVectorType() && !ValType->isRVVSizelessBuiltinType()) { Diag(DRE->getBeginLoc(), diag::err_nontemporal_builtin_must_be_pointer_intfltptr_or_vector) << PointerArg->getType() << PointerArg->getSourceRange(); @@ -7653,6 +7649,8 @@ static bool isValidOrderingForOp(int64_t Ordering, AtomicExpr::AtomicOp Op) { case AtomicExpr::AO__hip_atomic_load: case AtomicExpr::AO__atomic_load_n: case AtomicExpr::AO__atomic_load: + case AtomicExpr::AO__scoped_atomic_load_n: + case AtomicExpr::AO__scoped_atomic_load: return OrderingCABI != llvm::AtomicOrderingCABI::release && OrderingCABI != llvm::AtomicOrderingCABI::acq_rel; @@ -7661,6 +7659,8 @@ static bool isValidOrderingForOp(int64_t Ordering, AtomicExpr::AtomicOp Op) { case AtomicExpr::AO__hip_atomic_store: case AtomicExpr::AO__atomic_store: case AtomicExpr::AO__atomic_store_n: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_store_n: return OrderingCABI != llvm::AtomicOrderingCABI::consume && OrderingCABI != llvm::AtomicOrderingCABI::acquire && OrderingCABI != llvm::AtomicOrderingCABI::acq_rel; @@ -7737,13 +7737,19 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, Op <= AtomicExpr::AO__opencl_atomic_fetch_max; bool IsHIP = Op >= AtomicExpr::AO__hip_atomic_load && Op <= AtomicExpr::AO__hip_atomic_fetch_max; + bool IsScoped = Op >= AtomicExpr::AO__scoped_atomic_load && + Op <= AtomicExpr::AO__scoped_atomic_fetch_max; bool IsC11 = (Op >= AtomicExpr::AO__c11_atomic_init && Op <= AtomicExpr::AO__c11_atomic_fetch_min) || IsOpenCL; bool IsN = Op == AtomicExpr::AO__atomic_load_n || Op == AtomicExpr::AO__atomic_store_n || Op == AtomicExpr::AO__atomic_exchange_n || - Op == AtomicExpr::AO__atomic_compare_exchange_n; + Op == AtomicExpr::AO__atomic_compare_exchange_n || + Op == AtomicExpr::AO__scoped_atomic_load_n || + Op == AtomicExpr::AO__scoped_atomic_store_n || + Op == AtomicExpr::AO__scoped_atomic_exchange_n || + Op == AtomicExpr::AO__scoped_atomic_compare_exchange_n; // Bit mask for extra allowed value types other than integers for atomic // arithmetic operations. Add/sub allow pointer and floating point. Min/max // allow floating point. @@ -7764,10 +7770,12 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__opencl_atomic_load: case AtomicExpr::AO__hip_atomic_load: case AtomicExpr::AO__atomic_load_n: + case AtomicExpr::AO__scoped_atomic_load_n: Form = Load; break; case AtomicExpr::AO__atomic_load: + case AtomicExpr::AO__scoped_atomic_load: Form = LoadCopy; break; @@ -7776,12 +7784,18 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__hip_atomic_store: case AtomicExpr::AO__atomic_store: case AtomicExpr::AO__atomic_store_n: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_store_n: Form = Copy; break; case AtomicExpr::AO__atomic_fetch_add: case AtomicExpr::AO__atomic_fetch_sub: case AtomicExpr::AO__atomic_add_fetch: case AtomicExpr::AO__atomic_sub_fetch: + case AtomicExpr::AO__scoped_atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: case AtomicExpr::AO__c11_atomic_fetch_add: case AtomicExpr::AO__c11_atomic_fetch_sub: case AtomicExpr::AO__opencl_atomic_fetch_add: @@ -7795,6 +7809,10 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__atomic_fetch_min: case AtomicExpr::AO__atomic_max_fetch: case AtomicExpr::AO__atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: case AtomicExpr::AO__c11_atomic_fetch_max: case AtomicExpr::AO__c11_atomic_fetch_min: case AtomicExpr::AO__opencl_atomic_fetch_max: @@ -7822,6 +7840,14 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__atomic_or_fetch: case AtomicExpr::AO__atomic_xor_fetch: case AtomicExpr::AO__atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: Form = Arithmetic; break; @@ -7829,10 +7855,12 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__hip_atomic_exchange: case AtomicExpr::AO__opencl_atomic_exchange: case AtomicExpr::AO__atomic_exchange_n: + case AtomicExpr::AO__scoped_atomic_exchange_n: Form = Xchg; break; case AtomicExpr::AO__atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange: Form = GNUXchg; break; @@ -7847,12 +7875,15 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__atomic_compare_exchange: case AtomicExpr::AO__atomic_compare_exchange_n: + case AtomicExpr::AO__scoped_atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: Form = GNUCmpXchg; break; } unsigned AdjustedNumArgs = NumArgs[Form]; - if ((IsOpenCL || IsHIP) && Op != AtomicExpr::AO__opencl_atomic_init) + if ((IsOpenCL || IsHIP || IsScoped) && + Op != AtomicExpr::AO__opencl_atomic_init) ++AdjustedNumArgs; // Check we have the right number of arguments. if (Args.size() < AdjustedNumArgs) { diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp index 6169144ef1c2d4863a1294f9516f32695e1b9a22..143968b4ab0442fe017705f21dccbef066c164ab 100644 --- a/clang/lib/Sema/SemaCodeComplete.cpp +++ b/clang/lib/Sema/SemaCodeComplete.cpp @@ -10112,7 +10112,7 @@ void Sema::CodeCompleteIncludedFile(llvm::StringRef Dir, bool Angled) { const StringRef &Dirname = llvm::sys::path::filename(Dir); const bool isQt = Dirname.startswith("Qt") || Dirname == "ActiveQt"; const bool ExtensionlessHeaders = - IsSystem || isQt || Dir.endswith(".framework/Headers"); + IsSystem || isQt || Dir.ends_with(".framework/Headers"); std::error_code EC; unsigned Count = 0; for (auto It = FS.dir_begin(Dir, EC); diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index f12424d33b7da2d1c68880903fac404549127174..19d972ed8ab2d830ec9b099d3a820ff00561bc03 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -5971,7 +5971,7 @@ Sema::GetNameFromUnqualifiedId(const UnqualifiedId &Name) { diag::err_deduction_guide_name_not_class_template) << (int)getTemplateNameKindForDiagnostics(TN) << TN; if (Template) - Diag(Template->getLocation(), diag::note_template_decl_here); + NoteTemplateLocation(*Template); return DeclarationNameInfo(); } @@ -8914,7 +8914,7 @@ void Sema::CheckVariableDeclarationType(VarDecl *NewVD) { } } - if (T->isRVVType()) + if (T->isRVVSizelessBuiltinType()) checkRVVTypeSupport(T, NewVD->getLocation(), cast(CurContext)); } @@ -16221,7 +16221,9 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body, ActivePolicy = &WP; } - if (!IsInstantiation && FD && FD->isConstexpr() && !FD->isInvalidDecl() && + if (!IsInstantiation && FD && + (FD->isConstexpr() || FD->hasAttr()) && + !FD->isInvalidDecl() && !CheckConstexprFunctionDefinition(FD, CheckConstexprKind::Diagnose)) FD->setInvalidDecl(); diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index a345978bb8701ce151b8ecf97c0e686c235cf08d..59e456fd9f729837de00d86e5511db839d960c6d 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -7372,6 +7372,28 @@ static void handleDeclspecThreadAttr(Sema &S, Decl *D, const ParsedAttr &AL) { D->addAttr(::new (S.Context) ThreadAttr(S.Context, AL)); } +static void handleMSConstexprAttr(Sema &S, Decl *D, const ParsedAttr &AL) { + if (!S.getLangOpts().isCompatibleWithMSVC(LangOptions::MSVC2022_3)) { + S.Diag(AL.getLoc(), diag::warn_unknown_attribute_ignored) + << AL << AL.getRange(); + return; + } + auto *FD = cast(D); + if (FD->isConstexprSpecified() || FD->isConsteval()) { + S.Diag(AL.getLoc(), diag::err_ms_constexpr_cannot_be_applied) + << FD->isConsteval() << FD; + return; + } + if (auto *MD = dyn_cast(FD)) { + if (!S.getLangOpts().CPlusPlus20 && MD->isVirtual()) { + S.Diag(AL.getLoc(), diag::err_ms_constexpr_cannot_be_applied) + << /*virtual*/ 2 << MD; + return; + } + } + D->addAttr(::new (S.Context) MSConstexprAttr(S.Context, AL)); +} + static void handleAbiTagAttr(Sema &S, Decl *D, const ParsedAttr &AL) { SmallVector Tags; for (unsigned I = 0, E = AL.getNumArgs(); I != E; ++I) { @@ -9477,6 +9499,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL, case ParsedAttr::AT_Thread: handleDeclspecThreadAttr(S, D, AL); break; + case ParsedAttr::AT_MSConstexpr: + handleMSConstexprAttr(S, D, AL); + break; // HLSL attributes: case ParsedAttr::AT_HLSLNumThreads: diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp index 8fedf41d8424ad9f925c4e9ea9a9d7cac78ff3ad..36e53c684ac4dc3b012764202dfcd4c3e914ea54 100644 --- a/clang/lib/Sema/SemaDeclCXX.cpp +++ b/clang/lib/Sema/SemaDeclCXX.cpp @@ -11462,7 +11462,7 @@ bool Sema::CheckDeductionGuideDeclarator(Declarator &D, QualType &R, GuidedTemplateDecl->getDeclContext()->getRedeclContext())) { Diag(D.getIdentifierLoc(), diag::err_deduction_guide_wrong_scope) << GuidedTemplateDecl; - Diag(GuidedTemplateDecl->getLocation(), diag::note_template_decl_here); + NoteTemplateLocation(*GuidedTemplateDecl); } auto &DS = D.getMutableDeclSpec(); @@ -17879,6 +17879,8 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, LookupResult Previous(*this, NameInfo, LookupOrdinaryName, ForExternalRedeclaration); + bool isTemplateId = D.getName().getKind() == UnqualifiedIdKind::IK_TemplateId; + // There are five cases here. // - There's no scope specifier and we're in a local class. Only look // for functions declared in the immediately-enclosing block scope. @@ -17916,14 +17918,6 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, } adjustContextForLocalExternDecl(DC); - // C++ [class.friend]p6: - // A function can be defined in a friend declaration of a class if and - // only if the class is a non-local class (9.8), the function name is - // unqualified, and the function has namespace scope. - if (D.isFunctionDefinition()) { - Diag(NameInfo.getBeginLoc(), diag::err_friend_def_in_local_class); - } - // - There's no scope specifier, in which case we just go to the // appropriate scope and look for a function or function template // there as appropriate. @@ -17934,8 +17928,6 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, // elaborated-type-specifier, the lookup to determine whether // the entity has been previously declared shall not consider // any scopes outside the innermost enclosing namespace. - bool isTemplateId = - D.getName().getKind() == UnqualifiedIdKind::IK_TemplateId; // Find the appropriate context according to the above. DC = CurContext; @@ -17988,39 +17980,12 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, diag::warn_cxx98_compat_friend_is_member : diag::err_friend_is_member); - if (D.isFunctionDefinition()) { - // C++ [class.friend]p6: - // A function can be defined in a friend declaration of a class if and - // only if the class is a non-local class (9.8), the function name is - // unqualified, and the function has namespace scope. - // - // FIXME: We should only do this if the scope specifier names the - // innermost enclosing namespace; otherwise the fixit changes the - // meaning of the code. - SemaDiagnosticBuilder DB - = Diag(SS.getRange().getBegin(), diag::err_qualified_friend_def); - - DB << SS.getScopeRep(); - if (DC->isFileContext()) - DB << FixItHint::CreateRemoval(SS.getRange()); - SS.clear(); - } - // - There's a scope specifier that does not match any template // parameter lists, in which case we use some arbitrary context, // create a method or method template, and wait for instantiation. // - There's a scope specifier that does match some template // parameter lists, which we don't handle right now. } else { - if (D.isFunctionDefinition()) { - // C++ [class.friend]p6: - // A function can be defined in a friend declaration of a class if and - // only if the class is a non-local class (9.8), the function name is - // unqualified, and the function has namespace scope. - Diag(SS.getRange().getBegin(), diag::err_qualified_friend_def) - << SS.getScopeRep(); - } - DC = CurContext; assert(isa(DC) && "friend declaration not in class?"); } @@ -18105,6 +18070,38 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, else FD = cast(ND); + // C++ [class.friend]p6: + // A function may be defined in a friend declaration of a class if and + // only if the class is a non-local class, and the function name is + // unqualified. + if (D.isFunctionDefinition()) { + // Qualified friend function definition. + if (SS.isNotEmpty()) { + // FIXME: We should only do this if the scope specifier names the + // innermost enclosing namespace; otherwise the fixit changes the + // meaning of the code. + SemaDiagnosticBuilder DB = + Diag(SS.getRange().getBegin(), diag::err_qualified_friend_def); + + DB << SS.getScopeRep(); + if (DC->isFileContext()) + DB << FixItHint::CreateRemoval(SS.getRange()); + + // Friend function defined in a local class. + } else if (FunctionContainingLocalClass) { + Diag(NameInfo.getBeginLoc(), diag::err_friend_def_in_local_class); + + // Per [basic.pre]p4, a template-id is not a name. Therefore, if we have + // a template-id, the function name is not unqualified because these is + // no name. While the wording requires some reading in-between the + // lines, GCC, MSVC, and EDG all consider a friend function + // specialization definitions // to be de facto explicit specialization + // and diagnose them as such. + } else if (isTemplateId) { + Diag(NameInfo.getBeginLoc(), diag::err_friend_specialization_def); + } + } + // C++11 [dcl.fct.default]p4: If a friend declaration specifies a // default argument expression, that declaration shall be a definition // and shall be the only declaration of the function or function diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp index 50ee0a5acb5586a8c13904831b00eaf70e3ecb3d..5ca6b232df66a560b97751c73f3c6ff87cb978d5 100644 --- a/clang/lib/Sema/SemaInit.cpp +++ b/clang/lib/Sema/SemaInit.cpp @@ -4085,16 +4085,13 @@ static bool hasCopyOrMoveCtorParam(ASTContext &Ctx, return Ctx.hasSameUnqualifiedType(ParmT, ClassT); } -static OverloadingResult -ResolveConstructorOverload(Sema &S, SourceLocation DeclLoc, - MultiExprArg Args, - OverloadCandidateSet &CandidateSet, - QualType DestType, - DeclContext::lookup_result Ctors, - OverloadCandidateSet::iterator &Best, - bool CopyInitializing, bool AllowExplicit, - bool OnlyListConstructors, bool IsListInit, - bool SecondStepOfCopyInit = false) { +static OverloadingResult ResolveConstructorOverload( + Sema &S, SourceLocation DeclLoc, MultiExprArg Args, + OverloadCandidateSet &CandidateSet, QualType DestType, + DeclContext::lookup_result Ctors, OverloadCandidateSet::iterator &Best, + bool CopyInitializing, bool AllowExplicit, bool OnlyListConstructors, + bool IsListInit, bool RequireActualConstructor, + bool SecondStepOfCopyInit = false) { CandidateSet.clear(OverloadCandidateSet::CSK_InitByConstructor); CandidateSet.setDestAS(DestType.getQualifiers().getAddressSpace()); @@ -4157,7 +4154,7 @@ ResolveConstructorOverload(Sema &S, SourceLocation DeclLoc, // Note: SecondStepOfCopyInit is only ever true in this case when // evaluating whether to produce a C++98 compatibility warning. if (S.getLangOpts().CPlusPlus17 && Args.size() == 1 && - !SecondStepOfCopyInit) { + !RequireActualConstructor && !SecondStepOfCopyInit) { Expr *Initializer = Args[0]; auto *SourceRD = Initializer->getType()->getAsCXXRecordDecl(); if (SourceRD && S.isCompleteType(DeclLoc, Initializer->getType())) { @@ -4225,6 +4222,12 @@ static void TryConstructorInitialization(Sema &S, return; } + bool RequireActualConstructor = + !(Entity.getKind() != InitializedEntity::EK_Base && + Entity.getKind() != InitializedEntity::EK_Delegating && + Entity.getKind() != + InitializedEntity::EK_LambdaToBlockConversionBlockElement); + // C++17 [dcl.init]p17: // - If the initializer expression is a prvalue and the cv-unqualified // version of the source type is the same class as the class of the @@ -4234,11 +4237,7 @@ static void TryConstructorInitialization(Sema &S, // class or delegating to another constructor from a mem-initializer. // ObjC++: Lambda captured by the block in the lambda to block conversion // should avoid copy elision. - if (S.getLangOpts().CPlusPlus17 && - Entity.getKind() != InitializedEntity::EK_Base && - Entity.getKind() != InitializedEntity::EK_Delegating && - Entity.getKind() != - InitializedEntity::EK_LambdaToBlockConversionBlockElement && + if (S.getLangOpts().CPlusPlus17 && !RequireActualConstructor && UnwrappedArgs.size() == 1 && UnwrappedArgs[0]->isPRValue() && S.Context.hasSameUnqualifiedType(UnwrappedArgs[0]->getType(), DestType)) { // Convert qualifications if necessary. @@ -4286,11 +4285,10 @@ static void TryConstructorInitialization(Sema &S, // If the initializer list has no elements and T has a default constructor, // the first phase is omitted. if (!(UnwrappedArgs.empty() && S.LookupDefaultConstructor(DestRecordDecl))) - Result = ResolveConstructorOverload(S, Kind.getLocation(), Args, - CandidateSet, DestType, Ctors, Best, - CopyInitialization, AllowExplicit, - /*OnlyListConstructors=*/true, - IsListInit); + Result = ResolveConstructorOverload( + S, Kind.getLocation(), Args, CandidateSet, DestType, Ctors, Best, + CopyInitialization, AllowExplicit, + /*OnlyListConstructors=*/true, IsListInit, RequireActualConstructor); } // C++11 [over.match.list]p1: @@ -4300,11 +4298,10 @@ static void TryConstructorInitialization(Sema &S, // elements of the initializer list. if (Result == OR_No_Viable_Function) { AsInitializerList = false; - Result = ResolveConstructorOverload(S, Kind.getLocation(), UnwrappedArgs, - CandidateSet, DestType, Ctors, Best, - CopyInitialization, AllowExplicit, - /*OnlyListConstructors=*/false, - IsListInit); + Result = ResolveConstructorOverload( + S, Kind.getLocation(), UnwrappedArgs, CandidateSet, DestType, Ctors, + Best, CopyInitialization, AllowExplicit, + /*OnlyListConstructors=*/false, IsListInit, RequireActualConstructor); } if (Result) { Sequence.SetOverloadFailure( @@ -6778,6 +6775,7 @@ static ExprResult CopyObject(Sema &S, S, Loc, CurInitExpr, CandidateSet, T, Ctors, Best, /*CopyInitializing=*/false, /*AllowExplicit=*/true, /*OnlyListConstructors=*/false, /*IsListInit=*/false, + /*RequireActualConstructor=*/false, /*SecondStepOfCopyInit=*/true)) { case OR_Success: break; @@ -6920,6 +6918,7 @@ static void CheckCXX98CompatAccessibleCopy(Sema &S, S, Loc, CurInitExpr, CandidateSet, CurInitExpr->getType(), Ctors, Best, /*CopyInitializing=*/false, /*AllowExplicit=*/true, /*OnlyListConstructors=*/false, /*IsListInit=*/false, + /*RequireActualConstructor=*/false, /*SecondStepOfCopyInit=*/true); PartialDiagnostic Diag = S.PDiag(diag::warn_cxx98_compat_temp_copy) @@ -10573,7 +10572,7 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer( diag::err_deduced_non_class_template_specialization_type) << (int)getTemplateNameKindForDiagnostics(TemplateName) << TemplateName; if (auto *TD = TemplateName.getAsTemplateDecl()) - Diag(TD->getLocation(), diag::note_template_decl_here); + NoteTemplateLocation(*TD); return QualType(); } diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp index ca09b0481bcac762fc4b8d43bf40468e30096c75..4cc87c9fa765c4e544e1b06914a135d443ccc23c 100644 --- a/clang/lib/Sema/SemaLambda.cpp +++ b/clang/lib/Sema/SemaLambda.cpp @@ -1444,7 +1444,7 @@ void Sema::ActOnStartOfLambdaDefinition(LambdaIntroducer &Intro, for (const auto &Capture : Intro.Captures) { if (Capture.Id == TP->getIdentifier()) { Diag(Capture.Loc, diag::err_template_param_shadow) << Capture.Id; - Diag(TP->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*TP); } } } diff --git a/clang/lib/Sema/SemaRISCVVectorLookup.cpp b/clang/lib/Sema/SemaRISCVVectorLookup.cpp index 9a5aecf669a07dfb2d57d193c24bde5071435333..0d411fca0f9c8275489fb476c1850714f49d5b44 100644 --- a/clang/lib/Sema/SemaRISCVVectorLookup.cpp +++ b/clang/lib/Sema/SemaRISCVVectorLookup.cpp @@ -34,12 +34,6 @@ namespace { // Function definition of a RVV intrinsic. struct RVVIntrinsicDef { - /// Full function name with suffix, e.g. vadd_vv_i32m1. - std::string Name; - - /// Overloaded function name, e.g. vadd. - std::string OverloadName; - /// Mapping to which clang built-in function, e.g. __builtin_rvv_vadd. std::string BuiltinName; @@ -393,7 +387,7 @@ void RISCVIntrinsicManagerImpl::InitRVVIntrinsic( // Put into IntrinsicList. size_t Index = IntrinsicList.size(); - IntrinsicList.push_back({Name, OverloadedName, BuiltinName, Signature}); + IntrinsicList.push_back({BuiltinName, Signature}); // Creating mapping to Intrinsics. Intrinsics.insert({Name, Index}); diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp index 2b45aa5dff7be7cf8032d4ecf9e7de0ceddd015f..63348d27a8c94a1b4a24cb6fe4c4a4607a1655f9 100644 --- a/clang/lib/Sema/SemaStmt.cpp +++ b/clang/lib/Sema/SemaStmt.cpp @@ -1327,6 +1327,9 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch, } } + if (!TheDefaultStmt) + Diag(SwitchLoc, diag::warn_switch_default); + if (!HasDependentValue) { // If we don't have a default statement, check whether the // condition is constant. diff --git a/clang/lib/Sema/SemaStmtAttr.cpp b/clang/lib/Sema/SemaStmtAttr.cpp index eae1eaa2f9563d536c8d255f935fbf8ef803988a..725d8efe3828d659dbbeb58b91571a1453db50c3 100644 --- a/clang/lib/Sema/SemaStmtAttr.cpp +++ b/clang/lib/Sema/SemaStmtAttr.cpp @@ -397,6 +397,16 @@ static void CheckForDuplicateCodeAlignAttrs(Sema &S, } } +static Attr *handleMSConstexprAttr(Sema &S, Stmt *St, const ParsedAttr &A, + SourceRange Range) { + if (!S.getLangOpts().isCompatibleWithMSVC(LangOptions::MSVC2022_3)) { + S.Diag(A.getLoc(), diag::warn_unknown_attribute_ignored) + << A << A.getRange(); + return nullptr; + } + return ::new (S.Context) MSConstexprAttr(S.Context, A); +} + #define WANT_STMT_MERGE_LOGIC #include "clang/Sema/AttrParsedAttrImpl.inc" #undef WANT_STMT_MERGE_LOGIC @@ -600,6 +610,8 @@ static Attr *ProcessStmtAttribute(Sema &S, Stmt *St, const ParsedAttr &A, return handleUnlikely(S, St, A, Range); case ParsedAttr::AT_CodeAlign: return handleCodeAlignAttr(S, St, A); + case ParsedAttr::AT_MSConstexpr: + return handleMSConstexprAttr(S, St, A, Range); default: // N.B., ClangAttrEmitter.cpp emits a diagnostic helper that ensures a // declaration attribute is not written on a statement, but this code is diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index 09bbf14d39af5ad09f68859261d6a58d944c3745..f10abeaba0d4517ed5df60d4fbaae7919b7d829a 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -820,7 +820,6 @@ bool Sema::DiagnoseUninstantiableTemplate(SourceLocation PointOfInstantiation, if (!Complain || (PatternDef && PatternDef->isInvalidDecl())) return true; - std::optional Note; QualType InstantiationTy; if (TagDecl *TD = dyn_cast(Instantiation)) InstantiationTy = Context.getTypeDeclType(TD); @@ -838,25 +837,25 @@ bool Sema::DiagnoseUninstantiableTemplate(SourceLocation PointOfInstantiation, diag::err_explicit_instantiation_undefined_member) << /*member function*/ 1 << Instantiation->getDeclName() << Instantiation->getDeclContext(); - Note = diag::note_explicit_instantiation_here; + Diag(Pattern->getLocation(), diag::note_explicit_instantiation_here); } else { assert(isa(Instantiation) && "Must be a TagDecl!"); Diag(PointOfInstantiation, diag::err_implicit_instantiate_member_undefined) << InstantiationTy; - Note = diag::note_member_declared_at; + Diag(Pattern->getLocation(), diag::note_member_declared_at); } } else { if (isa(Instantiation)) { Diag(PointOfInstantiation, diag::err_explicit_instantiation_undefined_func_template) << Pattern; - Note = diag::note_explicit_instantiation_here; + Diag(Pattern->getLocation(), diag::note_explicit_instantiation_here); } else if (isa(Instantiation)) { Diag(PointOfInstantiation, diag::err_template_instantiate_undefined) << (TSK != TSK_ImplicitInstantiation) << InstantiationTy; - Note = diag::note_template_decl_here; + NoteTemplateLocation(*Pattern); } else { assert(isa(Instantiation) && "Must be a VarDecl!"); if (isa(Instantiation)) { @@ -869,11 +868,9 @@ bool Sema::DiagnoseUninstantiableTemplate(SourceLocation PointOfInstantiation, diag::err_explicit_instantiation_undefined_member) << /*static data member*/ 2 << Instantiation->getDeclName() << Instantiation->getDeclContext(); - Note = diag::note_explicit_instantiation_here; + Diag(Pattern->getLocation(), diag::note_explicit_instantiation_here); } } - if (Note) // Diagnostics were emitted. - Diag(Pattern->getLocation(), *Note); // In general, Instantiation isn't marked invalid to get more than one // error for multiple undefined instantiations. But the code that does @@ -898,8 +895,9 @@ void Sema::DiagnoseTemplateParameterShadow(SourceLocation Loc, Decl *PrevDecl) { // Make this a warning when MSVC compatibility is requested. unsigned DiagId = getLangOpts().MSVCCompat ? diag::ext_template_param_shadow : diag::err_template_param_shadow; - Diag(Loc, DiagId) << cast(PrevDecl)->getDeclName(); - Diag(PrevDecl->getLocation(), diag::note_template_param_here); + const auto *ND = cast(PrevDecl); + Diag(Loc, DiagId) << ND->getDeclName(); + NoteTemplateParameterLocation(*ND); } /// AdjustDeclIfTemplate - If the given decl happens to be a template, reset @@ -1714,6 +1712,8 @@ class ConstraintRefersToContainingTemplateChecker // Friend, likely because it was referred to without its template arguments. void CheckIfContainingRecord(const CXXRecordDecl *CheckingRD) { CheckingRD = CheckingRD->getMostRecentDecl(); + if (!CheckingRD->isTemplated()) + return; for (const DeclContext *DC = Friend->getLexicalDeclContext(); DC && !DC->isFileContext(); DC = DC->getParent()) @@ -4488,7 +4488,7 @@ static void checkMoreSpecializedThanPrimary(Sema &S, PartialSpecDecl *Partial) { << SFINAEArgString; } - S.Diag(Template->getLocation(), diag::note_template_decl_here); + S.NoteTemplateLocation(*Template); SmallVector PartialAC, TemplateAC; Template->getAssociatedConstraints(TemplateAC); Partial->getAssociatedConstraints(PartialAC); @@ -4966,8 +4966,7 @@ void Sema::diagnoseMissingTemplateArguments(TemplateName Name, Diag(Loc, diag::err_template_missing_args) << (int)getTemplateNameKindForDiagnostics(Name) << Name; if (TemplateDecl *TD = Name.getAsTemplateDecl()) { - Diag(TD->getLocation(), diag::note_template_decl_here) - << TD->getTemplateParameters()->getSourceRange(); + NoteTemplateLocation(*TD, TD->getTemplateParameters()->getSourceRange()); } } @@ -5319,7 +5318,7 @@ bool Sema::CheckTemplateTypeArgument( ? diag::ext_ms_template_type_arg_missing_typename : diag::err_template_arg_must_be_type_suggest) << FixItHint::CreateInsertion(Loc, "typename "); - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); // Recover by synthesizing a type using the location information that we // already have. @@ -5348,7 +5347,7 @@ bool Sema::CheckTemplateTypeArgument( // is not a type. SourceRange SR = AL.getSourceRange(); Diag(SR.getBegin(), diag::err_template_arg_must_be_type) << SR; - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return true; } @@ -5794,8 +5793,8 @@ bool Sema::CheckTemplateArgument( // therefore cannot be a non-type template argument. Diag(Arg.getLocation(), diag::err_template_arg_must_be_expr) << Arg.getSourceRange(); + NoteTemplateParameterLocation(*Param); - Diag(Param->getLocation(), diag::note_template_param_here); return true; case TemplateArgument::Type: { @@ -5815,7 +5814,7 @@ bool Sema::CheckTemplateArgument( Diag(SR.getBegin(), diag::err_template_arg_nontype_ambig) << SR << T; else Diag(SR.getBegin(), diag::err_template_arg_must_be_expr) << SR; - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return true; } @@ -5936,8 +5935,7 @@ static bool diagnoseMissingArgument(Sema &S, SourceLocation Loc, << /*not enough args*/0 << (int)S.getTemplateNameKindForDiagnostics(TemplateName(TD)) << TD; - S.Diag(TD->getLocation(), diag::note_template_decl_here) - << Params->getSourceRange(); + S.NoteTemplateLocation(*TD, Params->getSourceRange()); return true; } @@ -6003,8 +6001,7 @@ bool Sema::CheckTemplateArgumentList( << /*not enough args*/0 << (int)getTemplateNameKindForDiagnostics(TemplateName(Template)) << Template; - Diag(Template->getLocation(), diag::note_template_decl_here) - << Params->getSourceRange(); + NoteTemplateLocation(*Template, Params->getSourceRange()); return true; } } @@ -6036,7 +6033,7 @@ bool Sema::CheckTemplateArgumentList( diag::err_template_expansion_into_fixed_list) << (isa(Template) ? 1 : 0) << NewArgs[ArgIdx].getSourceRange(); - Diag((*Param)->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(**Param); return true; } @@ -6232,8 +6229,7 @@ bool Sema::CheckTemplateArgumentList( << (int)getTemplateNameKindForDiagnostics(TemplateName(Template)) << Template << SourceRange(NewArgs[ArgIdx].getLocation(), NewArgs.getRAngleLoc()); - Diag(Template->getLocation(), diag::note_template_decl_here) - << Params->getSourceRange(); + NoteTemplateLocation(*Template, Params->getSourceRange()); return true; } @@ -6645,7 +6641,7 @@ isNullPointerValueTemplateArgument(Sema &S, NonTypeTemplateParmDecl *Param, for (unsigned I = 0, N = Notes.size(); I != N; ++I) S.Diag(Notes[I].first, Notes[I].second); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return NPV_Error; } @@ -6671,7 +6667,7 @@ isNullPointerValueTemplateArgument(Sema &S, NonTypeTemplateParmDecl *Param, // then recover as if the types were correct. S.Diag(Arg->getExprLoc(), diag::err_template_arg_wrongtype_null_constant) << Arg->getType() << ParamType << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return NPV_NullPointer; } @@ -6681,7 +6677,7 @@ isNullPointerValueTemplateArgument(Sema &S, NonTypeTemplateParmDecl *Param, // message with the information we have here. S.Diag(Arg->getExprLoc(), diag::err_template_arg_invalid) << EvalResult.Val.getAsString(S.Context, ParamType); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return NPV_Error; } @@ -6693,7 +6689,7 @@ isNullPointerValueTemplateArgument(Sema &S, NonTypeTemplateParmDecl *Param, << ParamType << FixItHint::CreateInsertion(Arg->getBeginLoc(), Code) << FixItHint::CreateInsertion(S.getLocForEndOfToken(Arg->getEndLoc()), ")"); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return NPV_NullPointer; } @@ -6734,7 +6730,7 @@ static bool CheckTemplateArgumentIsCompatibleWithParameter( S.Diag(Arg->getBeginLoc(), diag::err_template_arg_ref_bind_ignores_quals) << ParamType << Arg->getType() << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } } @@ -6752,7 +6748,7 @@ static bool CheckTemplateArgumentIsCompatibleWithParameter( else S.Diag(Arg->getBeginLoc(), diag::err_template_arg_not_convertible) << ArgIn->getType() << ParamType << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } } @@ -6895,7 +6891,7 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction( if (!Entity) { S.Diag(Arg->getBeginLoc(), diag::err_template_arg_not_decl_ref) << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } @@ -6903,7 +6899,7 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction( if (isa(Entity) || isa(Entity)) { S.Diag(Arg->getBeginLoc(), diag::err_template_arg_field) << Entity << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } @@ -6912,7 +6908,7 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction( if (!Method->isStatic()) { S.Diag(Arg->getBeginLoc(), diag::err_template_arg_method) << Method << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } } @@ -6952,7 +6948,7 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction( if (Var->getType()->isReferenceType()) { S.Diag(Arg->getBeginLoc(), diag::err_template_arg_reference_var) << Var->getType() << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } @@ -6973,14 +6969,14 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction( ParamType.getNonReferenceType())) { S.Diag(AddrOpLoc, diag::err_template_arg_address_of_non_pointer) << ParamType; - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } S.Diag(AddrOpLoc, diag::err_template_arg_address_of_non_pointer) << ParamType << FixItHint::CreateRemoval(AddrOpLoc); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); ArgType = Entity->getType(); } @@ -7002,14 +6998,14 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction( if (!S.Context.hasSameUnqualifiedType(ArgType, ParamType)) { S.Diag(Arg->getBeginLoc(), diag::err_template_arg_not_address_of) << ParamType; - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } S.Diag(Arg->getBeginLoc(), diag::err_template_arg_not_address_of) << ParamType << FixItHint::CreateInsertion(Arg->getBeginLoc(), "&"); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); } } @@ -7125,7 +7121,7 @@ CheckTemplateArgumentPointerToMember(Sema &S, NonTypeTemplateParmDecl *Param, // We can't perform this conversion. S.Diag(ResultArg->getBeginLoc(), diag::err_template_arg_not_convertible) << ResultArg->getType() << ParamType << ResultArg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } @@ -7237,7 +7233,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, diag::err_non_type_template_parm_type_deduction_failure) << Param->getDeclName() << Param->getType() << Arg->getType() << Arg->getSourceRange(); - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return ExprError(); } } @@ -7247,7 +7243,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, // where the parameter type is deduced. ParamType = CheckNonTypeTemplateParameterType(ParamType, Arg->getExprLoc()); if (ParamType.isNull()) { - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return ExprError(); } } @@ -7283,7 +7279,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, Diag(StartLoc, diag::err_deduced_non_type_template_arg_type_mismatch) << Arg->getType() << ParamType.getUnqualifiedType(); - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return ExprError(); } @@ -7580,7 +7576,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, if (!ArgType->isIntegralOrEnumerationType()) { Diag(Arg->getBeginLoc(), diag::err_template_arg_not_integral_or_enumeral) << ArgType << Arg->getSourceRange(); - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return ExprError(); } else if (!Arg->isValueDependent()) { class TmplArgICEDiagnoser : public VerifyICEDiagnoser { @@ -7618,7 +7614,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, // We can't perform this conversion. Diag(Arg->getBeginLoc(), diag::err_template_arg_not_convertible) << Arg->getType() << ParamType << Arg->getSourceRange(); - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return ExprError(); } @@ -7664,7 +7660,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, Diag(Arg->getBeginLoc(), diag::warn_template_arg_negative) << toString(OldValue, 10) << toString(Value, 10) << Param->getType() << Arg->getSourceRange(); - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); } // Complain if we overflowed the template parameter's type. @@ -7679,7 +7675,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, Diag(Arg->getBeginLoc(), diag::warn_template_arg_too_large) << toString(OldValue, 10) << toString(Value, 10) << Param->getType() << Arg->getSourceRange(); - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); } } @@ -7806,7 +7802,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, case NPV_NotNullPointer: Diag(Arg->getExprLoc(), diag::err_template_arg_not_convertible) << Arg->getType() << ParamType; - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return ExprError(); case NPV_Error: @@ -7941,6 +7937,37 @@ bool Sema::CheckTemplateTemplateArgument(TemplateTemplateParmDecl *Param, Arg.getLocation()); } +static Sema::SemaDiagnosticBuilder noteLocation(Sema &S, const NamedDecl &Decl, + unsigned HereDiagID, + unsigned ExternalDiagID) { + if (Decl.getLocation().isValid()) + return S.Diag(Decl.getLocation(), HereDiagID); + + SmallString<128> Str; + llvm::raw_svector_ostream Out(Str); + PrintingPolicy PP = S.getPrintingPolicy(); + PP.TerseOutput = 1; + Decl.print(Out, PP); + return S.Diag(Decl.getLocation(), ExternalDiagID) << Out.str(); +} + +void Sema::NoteTemplateLocation(const NamedDecl &Decl, + std::optional ParamRange) { + SemaDiagnosticBuilder DB = + noteLocation(*this, Decl, diag::note_template_decl_here, + diag::note_template_decl_external); + if (ParamRange && ParamRange->isValid()) { + assert(Decl.getLocation().isValid() && + "Parameter range has location when Decl does not"); + DB << *ParamRange; + } +} + +void Sema::NoteTemplateParameterLocation(const NamedDecl &Decl) { + noteLocation(*this, Decl, diag::note_template_param_here, + diag::note_template_param_external); +} + /// Given a non-type template argument that refers to a /// declaration and the type of its corresponding non-type template /// parameter, produce an expression that properly refers to that @@ -8666,9 +8693,7 @@ static bool CheckNonTypeTemplatePartialSpecializationArgs( S.Diag(IsDefaultArgument ? TemplateNameLoc : ArgExpr->getBeginLoc(), diag::err_dependent_typed_non_type_arg_in_partial_spec) << Param->getType(); - S.Diag(Param->getLocation(), diag::note_template_param_here) - << (IsDefaultArgument ? ParamUseRange : SourceRange()) - << ParamUseRange; + S.NoteTemplateParameterLocation(*Param); return true; } } @@ -11248,7 +11273,7 @@ Sema::CheckTypenameType(ElaboratedTypeKeyword Keyword, else Diag(IILoc, diag::err_deduced_tst) << (int)getTemplateNameKindForDiagnostics(TemplateName(TD)); - Diag(TD->getLocation(), diag::note_template_decl_here); + NoteTemplateLocation(*TD); return QualType(); } return Context.getElaboratedType( diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp index 4ae027bd1bb6735d5bbbaaaf2db1a2cb2ae9e3bc..df6b40999e645c6825bae1fa591bff4e9bd53ed2 100644 --- a/clang/lib/Sema/SemaTemplateInstantiate.cpp +++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp @@ -810,6 +810,10 @@ void Sema::PrintInstantiationStack() { Diags.Report(Active->PointOfInstantiation, diag::note_template_nsdmi_here) << FD << Active->InstantiationRange; + } else if (ClassTemplateDecl *CTD = dyn_cast(D)) { + Diags.Report(Active->PointOfInstantiation, + diag::note_template_class_instantiation_here) + << CTD << Active->InstantiationRange; } else { Diags.Report(Active->PointOfInstantiation, diag::note_template_type_alias_instantiation_here) diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp index 56d133f20a2935170486e0a34c23989081a36cb2..83610503ed9b16b12089c54dc5e8766ef5272bbc 100644 --- a/clang/lib/Sema/SemaType.cpp +++ b/clang/lib/Sema/SemaType.cpp @@ -3792,7 +3792,7 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state, << Kind << Error << (int)SemaRef.getTemplateNameKindForDiagnostics(TN) << QualType(Deduced, 0) << AutoRange; if (auto *TD = TN.getAsTemplateDecl()) - SemaRef.Diag(TD->getLocation(), diag::note_template_decl_here); + SemaRef.NoteTemplateLocation(*TD); T = SemaRef.Context.IntTy; D.setInvalidType(true); diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index f22da838424b4155599c83341ac0f913f70a47f8..5b51ac40000d7a9894a25c3d7c2727f73dc4348a 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -2531,8 +2531,7 @@ InputFile ASTReader::getInputFile(ModuleFile &F, unsigned ID, bool Complain) { Overridden = false; } - OptionalFileEntryRefDegradesToFileEntryPtr File = OptionalFileEntryRef( - expectedToOptional(FileMgr.getFileRef(Filename, /*OpenFile=*/false))); + auto File = FileMgr.getOptionalFileRef(Filename, /*OpenFile=*/false); // For an overridden file, create a virtual file with the stored // size/timestamp. @@ -2559,7 +2558,8 @@ InputFile ASTReader::getInputFile(ModuleFile &F, unsigned ID, bool Complain) { // PCH. SourceManager &SM = getSourceManager(); // FIXME: Reject if the overrides are different. - if ((!Overridden && !Transient) && !SkipChecks && SM.isFileOverridden(File)) { + if ((!Overridden && !Transient) && !SkipChecks && + SM.isFileOverridden(*File)) { if (Complain) Error(diag::err_fe_pch_file_overridden, Filename); @@ -3152,7 +3152,7 @@ ASTReader::ReadControlBlock(ModuleFile &F, if (!bool(PP.getPreprocessorOpts().DisablePCHOrModuleValidation & DisableValidationForModuleKind::Module) && F.Kind != MK_ExplicitModule && F.Kind != MK_PrebuiltModule) { - auto BuildDir = PP.getFileManager().getDirectory(Blob); + auto BuildDir = PP.getFileManager().getOptionalDirectoryRef(Blob); if (!BuildDir || *BuildDir != M->Directory) { if (!canRecoverFromOutOfDate(F.FileName, ClientLoadCapabilities)) Diag(diag::err_imported_module_relocated) @@ -5786,7 +5786,7 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F, PartialDiagnostic(diag::err_module_file_conflict, ContextObj->DiagAllocator) << CurrentModule->getTopLevelModuleName() << CurFile->getName() - << F.File->getName(); + << F.File.getName(); return DiagnosticError::create(CurrentImportLoc, ConflictError); } } diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp index bc16cfc67a24f9fa45292ca92612aee7b5f1101f..7140a14aefbf9b188f70145a30258342eacc9bb1 100644 --- a/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/clang/lib/Serialization/ASTReaderDecl.cpp @@ -583,6 +583,9 @@ void ASTDeclReader::Visit(Decl *D) { } void ASTDeclReader::VisitDecl(Decl *D) { + BitsUnpacker DeclBits(Record.readInt()); + bool HasStandaloneLexicalDC = DeclBits.getNextBit(); + if (D->isTemplateParameter() || D->isTemplateParameterPack() || isa(D)) { // We don't want to deserialize the DeclContext of a template @@ -592,7 +595,8 @@ void ASTDeclReader::VisitDecl(Decl *D) { // return type of the function). Use the translation unit DeclContext as a // placeholder. GlobalDeclID SemaDCIDForTemplateParmDecl = readDeclID(); - GlobalDeclID LexicalDCIDForTemplateParmDecl = readDeclID(); + GlobalDeclID LexicalDCIDForTemplateParmDecl = + HasStandaloneLexicalDC ? readDeclID() : 0; if (!LexicalDCIDForTemplateParmDecl) LexicalDCIDForTemplateParmDecl = SemaDCIDForTemplateParmDecl; Reader.addPendingDeclContextInfo(D, @@ -601,7 +605,8 @@ void ASTDeclReader::VisitDecl(Decl *D) { D->setDeclContext(Reader.getContext().getTranslationUnitDecl()); } else { auto *SemaDC = readDeclAs(); - auto *LexicalDC = readDeclAs(); + auto *LexicalDC = + HasStandaloneLexicalDC ? readDeclAs() : nullptr; if (!LexicalDC) LexicalDC = SemaDC; // If the context is a class, we might not have actually merged it yet, in @@ -618,7 +623,6 @@ void ASTDeclReader::VisitDecl(Decl *D) { } D->setLocation(ThisDeclLoc); - BitsUnpacker DeclBits(Record.readInt()); D->InvalidDecl = DeclBits.getNextBit(); bool HasAttrs = DeclBits.getNextBit(); D->setImplicit(DeclBits.getNextBit()); @@ -765,7 +769,7 @@ ASTDeclReader::RedeclarableResult ASTDeclReader::VisitTagDecl(TagDecl *TD) { TD->setCompleteDefinitionRequired(TagDeclBits.getNextBit()); TD->setBraceRange(readSourceRange()); - switch (Record.readInt()) { + switch (TagDeclBits.getNextBits(/*Width=*/2)) { case 0: break; case 1: { // ExtInfo @@ -1089,7 +1093,8 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) { FD->setCachedLinkage((Linkage)FunctionDeclBits.getNextBits(/*Width=*/3)); FD->EndRangeLoc = readSourceLocation(); - FD->setDefaultLoc(readSourceLocation()); + if (FD->isExplicitlyDefaulted()) + FD->setDefaultLoc(readSourceLocation()); FD->ODRHash = Record.readInt(); FD->setHasODRHash(true); @@ -1703,7 +1708,7 @@ void ASTDeclReader::VisitParmVarDecl(ParmVarDecl *PD) { unsigned isObjCMethodParam = ParmVarDeclBits.getNextBit(); unsigned scopeDepth = ParmVarDeclBits.getNextBits(/*Width=*/7); unsigned scopeIndex = ParmVarDeclBits.getNextBits(/*Width=*/8); - unsigned declQualifier = Record.readInt(); + unsigned declQualifier = ParmVarDeclBits.getNextBits(/*Width=*/7); if (isObjCMethodParam) { assert(scopeDepth == 0); PD->setObjCMethodScopeInfo(scopeIndex); @@ -1716,7 +1721,9 @@ void ASTDeclReader::VisitParmVarDecl(ParmVarDecl *PD) { PD->ParmVarDeclBits.HasInheritedDefaultArg = ParmVarDeclBits.getNextBit(); if (ParmVarDeclBits.getNextBit()) // hasUninstantiatedDefaultArg. PD->setUninstantiatedDefaultArg(Record.readExpr()); - PD->ExplicitObjectParameterIntroducerLoc = Record.readSourceLocation(); + + if (ParmVarDeclBits.getNextBit()) // Valid explicit object parameter + PD->ExplicitObjectParameterIntroducerLoc = Record.readSourceLocation(); // FIXME: If this is a redeclaration of a function from another module, handle // inheritance of default arguments. diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp index d7d0c0e5bb21b47823e3152dd001ae20bf707256..b3a6f619372b4a77e1e134075bf8f9586d8c289f 100644 --- a/clang/lib/Serialization/ASTReaderStmt.cpp +++ b/clang/lib/Serialization/ASTReaderStmt.cpp @@ -108,7 +108,7 @@ namespace clang { /// The number of record fields required for the Expr class /// itself. - static const unsigned NumExprFields = NumStmtFields + 4; + static const unsigned NumExprFields = NumStmtFields + 2; /// Read and initialize a ExplicitTemplateArgumentList structure. void ReadTemplateKWAndArgsInfo(ASTTemplateKWAndArgsInfo &Args, @@ -524,9 +524,13 @@ void ASTStmtReader::VisitCapturedStmt(CapturedStmt *S) { void ASTStmtReader::VisitExpr(Expr *E) { VisitStmt(E); E->setType(Record.readType()); - E->setDependence(static_cast(Record.readInt())); - E->setValueKind(static_cast(Record.readInt())); - E->setObjectKind(static_cast(Record.readInt())); + BitsUnpacker ExprBits(Record.readInt()); + E->setDependence( + static_cast(ExprBits.getNextBits(/*Width=*/5))); + E->setValueKind( + static_cast(ExprBits.getNextBits(/*Width=*/2))); + E->setObjectKind( + static_cast(ExprBits.getNextBits(/*Width=*/3))); assert(Record.getIdx() == NumExprFields && "Incorrect expression field count"); } @@ -995,14 +999,19 @@ void ASTStmtReader::VisitOMPIteratorExpr(OMPIteratorExpr *E) { void ASTStmtReader::VisitCallExpr(CallExpr *E) { VisitExpr(E); - unsigned NumArgs = Record.readInt(); - bool HasFPFeatures = Record.readInt(); + + BitsUnpacker CallExprBits = Record.readInt(); + + unsigned NumArgs = CallExprBits.getNextBits(/*Width=*/16); + bool HasFPFeatures = CallExprBits.getNextBit(); + E->setADLCallKind( + static_cast(CallExprBits.getNextBit())); assert((NumArgs == E->getNumArgs()) && "Wrong NumArgs!"); E->setRParenLoc(readSourceLocation()); E->setCallee(Record.readSubExpr()); for (unsigned I = 0; I != NumArgs; ++I) E->setArg(I, Record.readSubExpr()); - E->setADLCallKind(static_cast(Record.readInt())); + if (HasFPFeatures) E->setStoredFPFeatures( FPOptionsOverride::getFromOpaqueInt(Record.readInt())); @@ -2013,14 +2022,15 @@ ASTStmtReader::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) { void ASTStmtReader::VisitOverloadExpr(OverloadExpr *E) { VisitExpr(E); - unsigned NumResults = Record.readInt(); - bool HasTemplateKWAndArgsInfo = Record.readInt(); + BitsUnpacker OverloadExprBits = Record.readInt(); + unsigned NumResults = OverloadExprBits.getNextBits(/*Width=*/14); + bool HasTemplateKWAndArgsInfo = OverloadExprBits.getNextBit(); assert((E->getNumDecls() == NumResults) && "Wrong NumResults!"); assert((E->hasTemplateKWAndArgsInfo() == HasTemplateKWAndArgsInfo) && "Wrong HasTemplateKWAndArgsInfo!"); if (HasTemplateKWAndArgsInfo) { - unsigned NumTemplateArgs = Record.readInt(); + unsigned NumTemplateArgs = OverloadExprBits.getNextBits(/*Width=*/14); ReadTemplateKWAndArgsInfo(*E->getTrailingASTTemplateKWAndArgsInfo(), E->getTrailingTemplateArgumentLoc(), NumTemplateArgs); @@ -3022,11 +3032,13 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { Record[ASTStmtReader::NumExprFields]); break; - case EXPR_CALL: - S = CallExpr::CreateEmpty( - Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], - /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty); + case EXPR_CALL: { + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto HasFPFeatures = CallExprBits.getNextBit(); + S = CallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); break; + } case EXPR_RECOVERY: S = RecoveryExpr::CreateEmpty( @@ -3764,17 +3776,23 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; } - case EXPR_CXX_OPERATOR_CALL: - S = CXXOperatorCallExpr::CreateEmpty( - Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], - /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty); + case EXPR_CXX_OPERATOR_CALL: { + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto HasFPFeatures = CallExprBits.getNextBit(); + S = CXXOperatorCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, + Empty); break; + } - case EXPR_CXX_MEMBER_CALL: - S = CXXMemberCallExpr::CreateEmpty( - Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], - /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty); + case EXPR_CXX_MEMBER_CALL: { + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto HasFPFeatures = CallExprBits.getNextBit(); + S = CXXMemberCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, + Empty); break; + } case EXPR_CXX_REWRITTEN_BINARY_OPERATOR: S = new (Context) CXXRewrittenBinaryOperator(Empty); @@ -3833,11 +3851,14 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { S = new (Context) BuiltinBitCastExpr(Empty); break; - case EXPR_USER_DEFINED_LITERAL: - S = UserDefinedLiteral::CreateEmpty( - Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], - /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty); + case EXPR_USER_DEFINED_LITERAL: { + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto HasFPFeatures = CallExprBits.getNextBit(); + S = UserDefinedLiteral::CreateEmpty(Context, NumArgs, HasFPFeatures, + Empty); break; + } case EXPR_CXX_STD_INITIALIZER_LIST: S = new (Context) CXXStdInitializerListExpr(Empty); @@ -3948,23 +3969,21 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { case EXPR_CXX_UNRESOLVED_MEMBER: S = UnresolvedMemberExpr::CreateEmpty( Context, - /*NumResults=*/Record[ASTStmtReader::NumExprFields], - /*HasTemplateKWAndArgsInfo=*/Record[ASTStmtReader::NumExprFields + 1], - /*NumTemplateArgs=*/ - Record[ASTStmtReader::NumExprFields + 1] - ? Record[ASTStmtReader::NumExprFields + 2] - : 0); + /*NumResults=*/Record[ASTStmtReader::NumExprFields] & ((1 << 14) - 1), + /*HasTemplateKWAndArgsInfo=*/ + (Record[ASTStmtReader::NumExprFields] >> 14) & (0x1), + /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields] >> 14 & + ((1 << 14) - 1)); break; case EXPR_CXX_UNRESOLVED_LOOKUP: S = UnresolvedLookupExpr::CreateEmpty( Context, - /*NumResults=*/Record[ASTStmtReader::NumExprFields], - /*HasTemplateKWAndArgsInfo=*/Record[ASTStmtReader::NumExprFields + 1], - /*NumTemplateArgs=*/ - Record[ASTStmtReader::NumExprFields + 1] - ? Record[ASTStmtReader::NumExprFields + 2] - : 0); + /*NumResults=*/Record[ASTStmtReader::NumExprFields] & ((1 << 14) - 1), + /*HasTemplateKWAndArgsInfo=*/ + (Record[ASTStmtReader::NumExprFields] >> 14) & (0x1), + /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields] >> 14 & + ((1 << 14) - 1)); break; case EXPR_TYPE_TRAIT: @@ -4024,11 +4043,14 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { S = new (Context) OpaqueValueExpr(Empty); break; - case EXPR_CUDA_KERNEL_CALL: - S = CUDAKernelCallExpr::CreateEmpty( - Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], - /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty); + case EXPR_CUDA_KERNEL_CALL: { + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto HasFPFeatures = CallExprBits.getNextBit(); + S = CUDAKernelCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, + Empty); break; + } case EXPR_ASTYPE: S = new (Context) AsTypeExpr(Empty); diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 6df815234e235fb50437308ae6dc75238123229c..91eb2af8f8ad6a55ce6b8d1b85b6111e93ce098c 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -1413,7 +1413,7 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context, // If we have calculated signature, there is no need to store // the size or timestamp. - Record.push_back(M.Signature ? 0 : M.File->getSize()); + Record.push_back(M.Signature ? 0 : M.File.getSize()); Record.push_back(M.Signature ? 0 : getTimestampForOutput(M.File)); llvm::append_range(Record, M.Signature); @@ -2182,8 +2182,8 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr, "Writing to AST an overridden file is not supported"); // The source location entry is a file. Emit input file ID. - assert(InputFileIDs[Content->OrigEntry] != 0 && "Missed file entry"); - Record.push_back(InputFileIDs[Content->OrigEntry]); + assert(InputFileIDs[*Content->OrigEntry] != 0 && "Missed file entry"); + Record.push_back(InputFileIDs[*Content->OrigEntry]); Record.push_back(getAdjustedNumCreatedFIDs(FID)); @@ -4695,7 +4695,7 @@ void ASTWriter::collectNonAffectingInputFiles() { if (!isModuleMap(File.getFileCharacteristic()) || AffectingModuleMaps.empty() || - AffectingModuleMaps.find(Cache->OrigEntry) != AffectingModuleMaps.end()) + llvm::is_contained(AffectingModuleMaps, *Cache->OrigEntry)) continue; IsSLocAffecting[I] = false; diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp index bf082e5b8eac61ace9021013f75161a5aa9bc960..43169b2befc687ed5731f6d189655ff324996b6d 100644 --- a/clang/lib/Serialization/ASTWriterDecl.cpp +++ b/clang/lib/Serialization/ASTWriterDecl.cpp @@ -320,13 +320,8 @@ void ASTDeclWriter::Visit(Decl *D) { } void ASTDeclWriter::VisitDecl(Decl *D) { - Record.AddDeclRef(cast_or_null(D->getDeclContext())); - if (D->getDeclContext() != D->getLexicalDeclContext()) - Record.AddDeclRef(cast_or_null(D->getLexicalDeclContext())); - else - Record.push_back(0); - BitsPacker DeclBits; + DeclBits.addBit(D->getDeclContext() != D->getLexicalDeclContext()); DeclBits.addBit(D->isInvalidDecl()); DeclBits.addBit(D->hasAttrs()); DeclBits.addBit(D->isImplicit()); @@ -337,6 +332,10 @@ void ASTDeclWriter::VisitDecl(Decl *D) { DeclBits.addBits((uint64_t)D->getModuleOwnershipKind(), /*BitWidth=*/3); Record.push_back(DeclBits); + Record.AddDeclRef(cast_or_null(D->getDeclContext())); + if (D->getDeclContext() != D->getLexicalDeclContext()) + Record.AddDeclRef(cast_or_null(D->getLexicalDeclContext())); + if (D->hasAttrs()) Record.AddAttributes(D->getAttrs()); @@ -450,19 +449,18 @@ void ASTDeclWriter::VisitTagDecl(TagDecl *D) { TagDeclBits.addBit(D->isEmbeddedInDeclarator()); TagDeclBits.addBit(D->isFreeStanding()); TagDeclBits.addBit(D->isCompleteDefinitionRequired()); + TagDeclBits.addBits( + D->hasExtInfo() ? 1 : (D->getTypedefNameForAnonDecl() ? 2 : 0), + /*BitWidth=*/2); Record.push_back(TagDeclBits); Record.AddSourceRange(D->getBraceRange()); if (D->hasExtInfo()) { - Record.push_back(1); Record.AddQualifierInfo(*D->getExtInfo()); } else if (auto *TD = D->getTypedefNameForAnonDecl()) { - Record.push_back(2); Record.AddDeclRef(TD); Record.AddIdentifierRef(TD->getDeclName().getAsIdentifierInfo()); - } else { - Record.push_back(0); } } @@ -702,7 +700,8 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) { Record.push_back(FunctionDeclBits); Record.AddSourceLocation(D->getEndLoc()); - Record.AddSourceLocation(D->getDefaultLoc()); + if (D->isExplicitlyDefaulted()) + Record.AddSourceLocation(D->getDefaultLoc()); Record.push_back(D->getODRHash()); @@ -1176,15 +1175,18 @@ void ASTDeclWriter::VisitParmVarDecl(ParmVarDecl *D) { ParmVarDeclBits.addBit(D->isObjCMethodParameter()); ParmVarDeclBits.addBits(D->getFunctionScopeDepth(), /*BitsWidth=*/7); ParmVarDeclBits.addBits(D->getFunctionScopeIndex(), /*BitsWidth=*/8); + // FIXME: stable encoding + ParmVarDeclBits.addBits(D->getObjCDeclQualifier(), /*BitsWidth=*/7); ParmVarDeclBits.addBit(D->isKNRPromoted()); ParmVarDeclBits.addBit(D->hasInheritedDefaultArg()); ParmVarDeclBits.addBit(D->hasUninstantiatedDefaultArg()); + ParmVarDeclBits.addBit(D->getExplicitObjectParamThisLoc().isValid()); Record.push_back(ParmVarDeclBits); - Record.push_back(D->getObjCDeclQualifier()); // FIXME: stable encoding if (D->hasUninstantiatedDefaultArg()) Record.AddStmt(D->getUninstantiatedDefaultArg()); - Record.AddSourceLocation(D->getExplicitObjectParamThisLoc()); + if (D->getExplicitObjectParamThisLoc().isValid()) + Record.AddSourceLocation(D->getExplicitObjectParamThisLoc()); Code = serialization::DECL_PARM_VAR; // If the assumptions about the DECL_PARM_VAR abbrev are true, use it. Here @@ -2038,13 +2040,12 @@ void ASTWriter::WriteDeclAbbrevs() { Abv = std::make_shared(); Abv->Add(BitCodeAbbrevOp(serialization::DECL_FIELD)); // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier @@ -2068,13 +2069,12 @@ void ASTWriter::WriteDeclAbbrevs() { Abv = std::make_shared(); Abv->Add(BitCodeAbbrevOp(serialization::DECL_OBJC_IVAR)); // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier @@ -2103,13 +2103,12 @@ void ASTWriter::WriteDeclAbbrevs() { // Redeclarable Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier @@ -2122,11 +2121,11 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // IdentifierNamespace Abv->Add(BitCodeAbbrevOp( BitCodeAbbrevOp::Fixed, - 7)); // Packed Tag Decl Bits: getTagKind, isCompleteDefinition, - // EmbeddedInDeclarator, IsFreeStanding, isCompleteDefinitionRequired + 9)); // Packed Tag Decl Bits: getTagKind, isCompleteDefinition, + // EmbeddedInDeclarator, IsFreeStanding, + // isCompleteDefinitionRequired, ExtInfoKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SourceLocation Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SourceLocation - Abv->Add(BitCodeAbbrevOp(0)); // ExtInfoKind // EnumDecl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // AddTypeRef Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // IntegerType @@ -2145,13 +2144,12 @@ void ASTWriter::WriteDeclAbbrevs() { // Redeclarable Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier @@ -2164,11 +2162,11 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // IdentifierNamespace Abv->Add(BitCodeAbbrevOp( BitCodeAbbrevOp::Fixed, - 7)); // Packed Tag Decl Bits: getTagKind, isCompleteDefinition, - // EmbeddedInDeclarator, IsFreeStanding, isCompleteDefinitionRequired + 9)); // Packed Tag Decl Bits: getTagKind, isCompleteDefinition, + // EmbeddedInDeclarator, IsFreeStanding, + // isCompleteDefinitionRequired, ExtInfoKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SourceLocation Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SourceLocation - Abv->Add(BitCodeAbbrevOp(0)); // ExtInfoKind // RecordDecl Abv->Add(BitCodeAbbrevOp( BitCodeAbbrevOp::Fixed, @@ -2194,13 +2192,12 @@ void ASTWriter::WriteDeclAbbrevs() { // Redeclarable Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier @@ -2221,10 +2218,9 @@ void ASTWriter::WriteDeclAbbrevs() { // ParmVarDecl Abv->Add(BitCodeAbbrevOp( BitCodeAbbrevOp::Fixed, - 19)); // Packed Parm Var Decl bits: IsObjCMethodParameter, ScopeDepth, - // ScopeIndex, KNRPromoted, HasInheritedDefaultArg - Abv->Add(BitCodeAbbrevOp(0)); // ObjCDeclQualifier - Abv->Add(BitCodeAbbrevOp(0)); // HasUninstantiatedDefaultArg + 27)); // Packed Parm Var Decl bits: IsObjCMethodParameter, ScopeDepth, + // ScopeIndex, ObjCDeclQualifier, KNRPromoted, + // HasInheritedDefaultArg, HasUninstantiatedDefaultArg // Type Source Info Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // TypeLoc @@ -2236,13 +2232,12 @@ void ASTWriter::WriteDeclAbbrevs() { // Redeclarable Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier @@ -2262,13 +2257,12 @@ void ASTWriter::WriteDeclAbbrevs() { // Redeclarable Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier @@ -2303,13 +2297,12 @@ void ASTWriter::WriteDeclAbbrevs() { // FIXME: Implement abbreviation for other template kinds. Abv->Add(BitCodeAbbrevOp(FunctionDecl::TK_NonTemplate)); // TemplateKind // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(DeclarationName::Identifier)); // NameKind @@ -2346,16 +2339,14 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); DeclCXXMethodAbbrev = Stream.EmitAbbrev(std::move(Abv)); - unsigned ExprDependenceBits = llvm::BitWidth; // Abbreviation for EXPR_DECL_REF Abv = std::make_shared(); Abv->Add(BitCodeAbbrevOp(serialization::EXPR_DECL_REF)); //Stmt // Expr Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, ExprDependenceBits)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetValueKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetObjectKind + // DependenceKind, ValueKind, ObjectKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); //DeclRefExpr Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //HasQualifier Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //GetDeclFound @@ -2374,9 +2365,8 @@ void ASTWriter::WriteDeclAbbrevs() { //Stmt // Expr Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, ExprDependenceBits)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetValueKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetObjectKind + // DependenceKind, ValueKind, ObjectKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); //Integer Literal Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Location Abv->Add(BitCodeAbbrevOp(32)); // Bit Width @@ -2389,9 +2379,8 @@ void ASTWriter::WriteDeclAbbrevs() { //Stmt // Expr Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, ExprDependenceBits)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetValueKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetObjectKind + // DependenceKind, ValueKind, ObjectKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); //Character Literal Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // getValue Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Location @@ -2404,9 +2393,8 @@ void ASTWriter::WriteDeclAbbrevs() { // Stmt // Expr Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, ExprDependenceBits)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetValueKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetObjectKind + // DependenceKind, ValueKind, ObjectKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); // CastExpr Abv->Add(BitCodeAbbrevOp(0)); // PathSize Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // HasFPFeatures diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp index 59be6828fafabf6ac17cbf155e1826dea560ac97..8524484ea8a0b98e559bdf0b20fb8d5f2fae4799 100644 --- a/clang/lib/Serialization/ASTWriterStmt.cpp +++ b/clang/lib/Serialization/ASTWriterStmt.cpp @@ -549,9 +549,14 @@ void ASTStmtWriter::VisitCapturedStmt(CapturedStmt *S) { void ASTStmtWriter::VisitExpr(Expr *E) { VisitStmt(E); Record.AddTypeRef(E->getType()); - Record.push_back(E->getDependence()); - Record.push_back(E->getValueKind()); - Record.push_back(E->getObjectKind()); + + BitsPacker ExprBits; + + ExprBits.addBits(E->getDependence(), /*BitsWidth=*/5); + ExprBits.addBits(E->getValueKind(), /*BitsWidth=*/2); + ExprBits.addBits(E->getObjectKind(), /*BitsWidth=*/3); + + Record.push_back(ExprBits); } void ASTStmtWriter::VisitConstantExpr(ConstantExpr *E) { @@ -866,14 +871,20 @@ void ASTStmtWriter::VisitOMPIteratorExpr(OMPIteratorExpr *E) { void ASTStmtWriter::VisitCallExpr(CallExpr *E) { VisitExpr(E); - Record.push_back(E->getNumArgs()); - Record.push_back(E->hasStoredFPFeatures()); + + BitsPacker CallExprBits; + // 16 bits should be sufficient to store the number args; + CallExprBits.addBits(E->getNumArgs(), /*BitsWidth=*/16); + CallExprBits.addBit(E->hasStoredFPFeatures()); + CallExprBits.addBit(static_cast(E->getADLCallKind())); + Record.push_back(CallExprBits); + Record.AddSourceLocation(E->getRParenLoc()); Record.AddStmt(E->getCallee()); for (CallExpr::arg_iterator Arg = E->arg_begin(), ArgEnd = E->arg_end(); Arg != ArgEnd; ++Arg) Record.AddStmt(*Arg); - Record.push_back(static_cast(E->getADLCallKind())); + if (E->hasStoredFPFeatures()) Record.push_back(E->getFPFeatures().getAsOpaqueInt()); Code = serialization::EXPR_CALL; @@ -1938,14 +1949,19 @@ ASTStmtWriter::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) { void ASTStmtWriter::VisitOverloadExpr(OverloadExpr *E) { VisitExpr(E); - Record.push_back(E->getNumDecls()); - Record.push_back(E->hasTemplateKWAndArgsInfo()); + BitsPacker OverloadExprBits; + // 14 Bits should enough to store the number of decls. + OverloadExprBits.addBits(E->getNumDecls(), /*BitWidth=*/14); + OverloadExprBits.addBit(E->hasTemplateKWAndArgsInfo()); if (E->hasTemplateKWAndArgsInfo()) { const ASTTemplateKWAndArgsInfo &ArgInfo = *E->getTrailingASTTemplateKWAndArgsInfo(); - Record.push_back(ArgInfo.NumTemplateArgs); + // 14 Bits should enough to store the number of template args. + OverloadExprBits.addBits(ArgInfo.NumTemplateArgs, /*BitWidth=*/14); + Record.push_back(OverloadExprBits); AddTemplateKWAndArgsInfo(ArgInfo, E->getTrailingTemplateArgumentLoc()); - } + } else + Record.push_back(OverloadExprBits); for (OverloadExpr::decls_iterator OvI = E->decls_begin(), OvE = E->decls_end(); diff --git a/clang/lib/Serialization/GlobalModuleIndex.cpp b/clang/lib/Serialization/GlobalModuleIndex.cpp index fb80a1998d0efe90708dbcd7ca642c4f26cbec28..dd4fc3e009050f756b8a3df620d3d47053c095b1 100644 --- a/clang/lib/Serialization/GlobalModuleIndex.cpp +++ b/clang/lib/Serialization/GlobalModuleIndex.cpp @@ -342,8 +342,8 @@ bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) { // If the size and modification time match what we expected, record this // module file. bool Failed = true; - if (File->File->getSize() == Info.Size && - File->File->getModificationTime() == Info.ModTime) { + if (File->File.getSize() == Info.Size && + File->File.getModificationTime() == Info.ModTime) { Info.File = File; ModulesByFile[File] = Known->second; diff --git a/clang/lib/Serialization/ModuleManager.cpp b/clang/lib/Serialization/ModuleManager.cpp index de4cd3d05853ac0f31d27baf4ee046b16a97f9bf..51b6429412960efcc48fbfa846c80612a1391f49 100644 --- a/clang/lib/Serialization/ModuleManager.cpp +++ b/clang/lib/Serialization/ModuleManager.cpp @@ -52,8 +52,8 @@ ModuleFile *ModuleManager::lookupByFileName(StringRef Name) const { ModuleFile *ModuleManager::lookupByModuleName(StringRef Name) const { if (const Module *Mod = HeaderSearchInfo.getModuleMap().findModule(Name)) - if (const FileEntry *File = Mod->getASTFile()) - return lookup(File); + if (OptionalFileEntryRef File = Mod->getASTFile()) + return lookup(*File); return nullptr; } @@ -108,7 +108,7 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, // Look for the file entry. This only fails if the expected size or // modification time differ. - OptionalFileEntryRefDegradesToFileEntryPtr Entry; + OptionalFileEntryRef Entry; if (Type == MK_ExplicitModule || Type == MK_PrebuiltModule) { // If we're not expecting to pull this file out of the module cache, it // might have a different mtime due to being moved across filesystems in @@ -123,7 +123,7 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, return OutOfDate; } - if (!Entry && FileName != "-") { + if (!Entry) { ErrorStr = "module file not found"; return Missing; } @@ -150,7 +150,7 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, }; // Check whether we already loaded this module, before - if (ModuleFile *ModuleEntry = Modules.lookup(Entry)) { + if (ModuleFile *ModuleEntry = Modules.lookup(*Entry)) { if (implicitModuleNamesMatch(Type, ModuleEntry, *Entry)) { // Check the stored signature. if (checkSignature(ModuleEntry->Signature, ExpectedSignature, ErrorStr)) @@ -163,10 +163,9 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, } // Allocate a new module. - auto NewModule = std::make_unique(Type, Generation); + auto NewModule = std::make_unique(Type, *Entry, Generation); NewModule->Index = Chain.size(); NewModule->FileName = FileName.str(); - NewModule->File = Entry; NewModule->ImportLoc = ImportLoc; NewModule->InputFilesValidationTimestamp = 0; @@ -198,21 +197,15 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, Entry->closeFile(); return OutOfDate; } else { - // Open the AST file. - llvm::ErrorOr> Buf((std::error_code())); - if (FileName == "-") { - Buf = llvm::MemoryBuffer::getSTDIN(); - } else { - // Get a buffer of the file and close the file descriptor when done. - // The file is volatile because in a parallel build we expect multiple - // compiler processes to use the same module file rebuilding it if needed. - // - // RequiresNullTerminator is false because module files don't need it, and - // this allows the file to still be mmapped. - Buf = FileMgr.getBufferForFile(*NewModule->File, - /*IsVolatile=*/true, - /*RequiresNullTerminator=*/false); - } + // Get a buffer of the file and close the file descriptor when done. + // The file is volatile because in a parallel build we expect multiple + // compiler processes to use the same module file rebuilding it if needed. + // + // RequiresNullTerminator is false because module files don't need it, and + // this allows the file to still be mmapped. + auto Buf = FileMgr.getBufferForFile(NewModule->File, + /*IsVolatile=*/true, + /*RequiresNullTerminator=*/false); if (!Buf) { ErrorStr = Buf.getError().message(); @@ -232,7 +225,7 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, return OutOfDate; // We're keeping this module. Store it everywhere. - Module = Modules[Entry] = NewModule.get(); + Module = Modules[*Entry] = NewModule.get(); updateModuleImports(*NewModule, ImportedBy, ImportLoc); @@ -441,22 +434,19 @@ void ModuleManager::visit(llvm::function_ref Visitor, bool ModuleManager::lookupModuleFile(StringRef FileName, off_t ExpectedSize, time_t ExpectedModTime, OptionalFileEntryRef &File) { - File = std::nullopt; - if (FileName == "-") + if (FileName == "-") { + File = expectedToOptional(FileMgr.getSTDIN()); return false; + } // Open the file immediately to ensure there is no race between stat'ing and // opening the file. - OptionalFileEntryRef FileOrErr = - expectedToOptional(FileMgr.getFileRef(FileName, /*OpenFile=*/true, - /*CacheFailure=*/false)); - if (!FileOrErr) - return false; - - File = *FileOrErr; + File = FileMgr.getOptionalFileRef(FileName, /*OpenFile=*/true, + /*CacheFailure=*/false); - if ((ExpectedSize && ExpectedSize != File->getSize()) || - (ExpectedModTime && ExpectedModTime != File->getModificationTime())) + if (File && + ((ExpectedSize && ExpectedSize != File->getSize()) || + (ExpectedModTime && ExpectedModTime != File->getModificationTime()))) // Do not destroy File, as it may be referenced. If we need to rebuild it, // it will be destroyed by removeModules. return true; diff --git a/clang/lib/StaticAnalyzer/Checkers/BitwiseShiftChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BitwiseShiftChecker.cpp index d4aa9fa1339f47cacd307e01f8a61e30fa254807..339927c165fe000ff1078c4bb8b907e61bc341e9 100644 --- a/clang/lib/StaticAnalyzer/Checkers/BitwiseShiftChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/BitwiseShiftChecker.cpp @@ -344,7 +344,7 @@ BitwiseShiftValidator::createBugReport(StringRef ShortMsg, StringRef Msg) const } // anonymous namespace class BitwiseShiftChecker : public Checker> { - mutable std::unique_ptr BTPtr; + BugType BT{this, "Bitwise shift", "Suspicious operation"}; public: void checkPreStmt(const BinaryOperator *B, CheckerContext &Ctx) const { @@ -353,11 +353,7 @@ public: if (Op != BO_Shl && Op != BO_Shr) return; - if (!BTPtr) - BTPtr = std::make_unique(this, "Bitwise shift", - "Suspicious operation"); - - BitwiseShiftValidator(B, Ctx, *BTPtr, Pedantic).run(); + BitwiseShiftValidator(B, Ctx, BT, Pedantic).run(); } bool Pedantic = false; diff --git a/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp index 5844f4399100183cc224688f3560ee6596581b87..14433d06c2d04eb2dc6e9e2f4d13b56b36f51f17 100644 --- a/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp @@ -22,10 +22,12 @@ #include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h" #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" +#include "llvm/Support/FormatVariadic.h" #include using namespace clang; using namespace ento; +using llvm::formatv; namespace { // This evaluator checks two SVals for equality. The first SVal is provided via @@ -87,17 +89,22 @@ void EnumCastOutOfRangeChecker::reportWarning(CheckerContext &C, EnumValueCastOutOfRange.reset( new BugType(this, "Enum cast out of range")); - llvm::SmallString<128> Msg{"The value provided to the cast expression is " - "not in the valid range of values for "}; - StringRef EnumName{E->getName()}; - if (EnumName.empty()) { - Msg += "the enum"; - } else { - Msg += '\''; - Msg += EnumName; - Msg += '\''; + std::string ValueStr = "", NameStr = "the enum"; + + // Try to add details to the message: + const auto ConcreteValue = + C.getSVal(CE->getSubExpr()).getAs(); + if (ConcreteValue) { + ValueStr = formatv(" '{0}'", ConcreteValue->getValue()); + } + if (StringRef EnumName{E->getName()}; !EnumName.empty()) { + NameStr = formatv("'{0}'", EnumName); } + std::string Msg = formatv("The value{0} provided to the cast expression is " + "not in the valid range of values for {1}", + ValueStr, NameStr); + auto BR = std::make_unique(*EnumValueCastOutOfRange, Msg, N); bugreporter::trackExpressionValue(N, CE->getSubExpr(), *BR); diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index 1058ddb8254cde9246e855a27bef64bd69643a09..f65da413bb87c35d46ceb76d5daf1d085d7d27d7 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -521,7 +521,7 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { serialization::ModuleFile *MF = MDC.ScanInstance.getASTReader()->getModuleManager().lookup( - M->getASTFile()); + *M->getASTFile()); MDC.ScanInstance.getASTReader()->visitInputFileInfos( *MF, /*IncludeSystem=*/true, [&](const serialization::InputFileInfo &IFI, bool IsSystem) { diff --git a/clang/test/AST/HLSL/RWBuffer-AST.hlsl b/clang/test/AST/HLSL/RWBuffer-AST.hlsl index 9dd9244b73eedf3443a9af9666bead19c60fabf9..c1613520a146c8c4efbb7a084b7049fabf9c08a3 100644 --- a/clang/test/AST/HLSL/RWBuffer-AST.hlsl +++ b/clang/test/AST/HLSL/RWBuffer-AST.hlsl @@ -13,8 +13,6 @@ // EMPTY: ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <> implicit RWBuffer // EMPTY-NEXT: TemplateTypeParmDecl 0x{{[0-9A-Fa-f]+}} <> class depth 0 index 0 element_type -// EMPTY-NEXT: TemplateArgument type 'float' -// EMPTY-NEXT: BuiltinType 0x{{[0-9A-Fa-f]+}} 'float' // EMPTY-NEXT: CXXRecordDecl 0x{{[0-9A-Fa-f]+}} <> implicit class RWBuffer // EMPTY-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <> Implicit final @@ -33,8 +31,6 @@ RWBuffer Buffer; // CHECK: ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <> implicit RWBuffer // CHECK-NEXT: TemplateTypeParmDecl 0x{{[0-9A-Fa-f]+}} <> class depth 0 index 0 element_type -// CHECK-NEXT: TemplateArgument type 'float' -// CHECK-NEXT: BuiltinType 0x{{[0-9A-Fa-f]+}} 'float' // CHECK-NEXT: CXXRecordDecl 0x{{[0-9A-Fa-f]+}} <> implicit class RWBuffer definition // CHECK: FinalAttr 0x{{[0-9A-Fa-f]+}} <> Implicit final diff --git a/clang/test/AST/HLSL/pch.hlsl b/clang/test/AST/HLSL/pch.hlsl index 74254fedf27cb5bc1b4106c82e9c472325d47262..27fae8f499daac737e0c59d542a1cdf5293aeabc 100644 --- a/clang/test/AST/HLSL/pch.hlsl +++ b/clang/test/AST/HLSL/pch.hlsl @@ -6,7 +6,7 @@ // Make sure PCH works by using function declared in PCH header and declare a RWBuffer in current file. // CHECK:FunctionDecl 0x[[FOO:[0-9a-f]+]] <{{.*}}:2:1, line:4:1> line:2:8 imported used foo 'float2 (float2, float2)' -// CHECK:VarDecl 0x{{[0-9a-f]+}} <{{.*}}:10:1, col:23> col:23 Buffer 'hlsl::RWBuffer':'hlsl::RWBuffer<>' +// CHECK:VarDecl 0x{{[0-9a-f]+}} <{{.*}}:10:1, col:23> col:23 Buffer 'hlsl::RWBuffer' hlsl::RWBuffer Buffer; float2 bar(float2 a, float2 b) { diff --git a/clang/test/AST/HLSL/pch_with_buf.hlsl b/clang/test/AST/HLSL/pch_with_buf.hlsl index 4e657606cbcb124ec727b294e2baa4e8a3d06f85..e8eae533af6294046dd3df8c3f074b72f9f035e7 100644 --- a/clang/test/AST/HLSL/pch_with_buf.hlsl +++ b/clang/test/AST/HLSL/pch_with_buf.hlsl @@ -5,9 +5,9 @@ // Make sure PCH works by using function declared in PCH header. // CHECK:FunctionDecl 0x[[FOO:[0-9a-f]+]] <{{.*}}:2:1, line:4:1> line:2:8 imported used foo 'float2 (float2, float2)' // Make sure buffer defined in PCH works. -// CHECK:VarDecl 0x{{[0-9a-f]+}} col:17 imported Buf 'RWBuffer':'hlsl::RWBuffer<>' +// CHECK:VarDecl 0x{{[0-9a-f]+}} col:17 imported Buf 'RWBuffer' // Make sure declare a RWBuffer in current file works. -// CHECK:VarDecl 0x{{[0-9a-f]+}} <{{.*}}:11:1, col:23> col:23 Buf2 'hlsl::RWBuffer':'hlsl::RWBuffer<>' +// CHECK:VarDecl 0x{{[0-9a-f]+}} <{{.*}}:11:1, col:23> col:23 Buf2 'hlsl::RWBuffer' hlsl::RWBuffer Buf2; float2 bar(float2 a, float2 b) { diff --git a/clang/test/AST/HLSL/resource_binding_attr.hlsl b/clang/test/AST/HLSL/resource_binding_attr.hlsl index 6836071971111eafe1db21ddf295b4a514ed63d6..71900f2dbda5503ef2c4fe5fc579faeddd5dca83 100644 --- a/clang/test/AST/HLSL/resource_binding_attr.hlsl +++ b/clang/test/AST/HLSL/resource_binding_attr.hlsl @@ -23,15 +23,15 @@ float foo() { return a + b; } -// CHECK: VarDecl 0x{{[0-9a-f]+}} <{{.*}}> col:17 UAV 'RWBuffer':'hlsl::RWBuffer<>' callinit -// CHECK-NEXT:-CXXConstructExpr 0x{{[0-9a-f]+}} 'RWBuffer':'hlsl::RWBuffer<>' 'void ()' +// CHECK: VarDecl 0x{{[0-9a-f]+}} <{{.*}}> col:17 UAV 'RWBuffer':'hlsl::RWBuffer' callinit +// CHECK-NEXT:-CXXConstructExpr 0x{{[0-9a-f]+}} 'RWBuffer':'hlsl::RWBuffer' 'void ()' // CHECK-NEXT:-HLSLResourceBindingAttr 0x{{[0-9a-f]+}} "u3" "space0" RWBuffer UAV : register(u3); -// CHECK: -VarDecl 0x{{[0-9a-f]+}} <{{.*}}> col:17 UAV1 'RWBuffer':'hlsl::RWBuffer<>' callinit -// CHECK-NEXT:-CXXConstructExpr 0x{{[0-9a-f]+}} 'RWBuffer':'hlsl::RWBuffer<>' 'void ()' +// CHECK: -VarDecl 0x{{[0-9a-f]+}} <{{.*}}> col:17 UAV1 'RWBuffer':'hlsl::RWBuffer' callinit +// CHECK-NEXT:-CXXConstructExpr 0x{{[0-9a-f]+}} 'RWBuffer':'hlsl::RWBuffer' 'void ()' // CHECK-NEXT:-HLSLResourceBindingAttr 0x{{[0-9a-f]+}} "u2" "space0" -// CHECK-NEXT:-VarDecl 0x{{[0-9a-f]+}} col:38 UAV2 'RWBuffer':'hlsl::RWBuffer<>' callinit -// CHECK-NEXT:-CXXConstructExpr 0x{{[0-9a-f]+}} 'RWBuffer':'hlsl::RWBuffer<>' 'void ()' +// CHECK-NEXT:-VarDecl 0x{{[0-9a-f]+}} col:38 UAV2 'RWBuffer':'hlsl::RWBuffer' callinit +// CHECK-NEXT:-CXXConstructExpr 0x{{[0-9a-f]+}} 'RWBuffer':'hlsl::RWBuffer' 'void ()' // CHECK-NEXT:-HLSLResourceBindingAttr 0x{{[0-9a-f]+}} "u4" "space0" RWBuffer UAV1 : register(u2), UAV2 : register(u4); diff --git a/clang/test/AST/Interp/builtin-functions.cpp b/clang/test/AST/Interp/builtin-functions.cpp index 0726dab37cb4eb0ea3df8b8a364eda98704fa32a..35a1f9a75092a05076ebd817986839355b39884b 100644 --- a/clang/test/AST/Interp/builtin-functions.cpp +++ b/clang/test/AST/Interp/builtin-functions.cpp @@ -331,3 +331,11 @@ namespace bitreverse { char bitreverse3[__builtin_bitreverse32(0x12345678) == 0x1E6A2C48 ? 1 : -1]; char bitreverse4[__builtin_bitreverse64(0x0123456789ABCDEFULL) == 0xF7B3D591E6A2C480 ? 1 : -1]; } + +namespace expect { + constexpr int a() { + return 12; + } + static_assert(__builtin_expect(a(),1) == 12, ""); + static_assert(__builtin_expect_with_probability(a(), 1, 1.0) == 12, ""); +} diff --git a/clang/test/AST/Interp/floats.cpp b/clang/test/AST/Interp/floats.cpp index e17167f5bf6dbbf66da24ec3edd3704fded2c4d1..45c31c759e47fc6e33d6d44f4e5203ec110213ac 100644 --- a/clang/test/AST/Interp/floats.cpp +++ b/clang/test/AST/Interp/floats.cpp @@ -39,6 +39,10 @@ constexpr float m = 5.0f / 0.0f; // ref-error {{must be initialized by a constan static_assert(~2.0f == 3, ""); // ref-error {{invalid argument type 'float' to unary expression}} \ // expected-error {{invalid argument type 'float' to unary expression}} + +typedef int tdb[(long long)4e20]; //expected-error {{variable length}} \ + //ref-error {{variable length}} + /// Initialized by a double. constexpr float df = 0.0; /// The other way around. diff --git a/clang/test/AST/ms-constexpr.cpp b/clang/test/AST/ms-constexpr.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e85af8494f3344a0ac42641ede76f1cd28551bb1 --- /dev/null +++ b/clang/test/AST/ms-constexpr.cpp @@ -0,0 +1,28 @@ +// RUN: %clang_cc1 -fms-compatibility -fms-compatibility-version=19.33 -std=c++20 -ast-dump -verify %s | FileCheck %s +// expected-no-diagnostics + +// CHECK: used f1 'bool ()' +// CHECK: MSConstexprAttr 0x{{[0-9a-f]+}} +[[msvc::constexpr]] bool f1() { return true; } + +// CHECK: used constexpr f2 'bool ()' +// CHECK-NEXT: CompoundStmt 0x{{[0-9a-f]+}} +// CHECK-NEXT: AttributedStmt 0x{{[0-9a-f]+}} +// CHECK-NEXT: MSConstexprAttr 0x{{[0-9a-f]+}} +// CHECK-NEXT: ReturnStmt 0x{{[0-9a-f]+}} +constexpr bool f2() { [[msvc::constexpr]] return f1(); } +static_assert(f2()); + +struct S1 { + // CHECK: used vm 'bool ()' virtual + // CHECK: MSConstexprAttr 0x{{[0-9a-f]+}} + [[msvc::constexpr]] virtual bool vm() { return true; } + + // CHECK: used constexpr cm 'bool ()' + // CHECK-NEXT: CompoundStmt 0x{{[0-9a-f]+}} + // CHECK-NEXT: AttributedStmt 0x{{[0-9a-f]+}} + // CHECK-NEXT: MSConstexprAttr 0x{{[0-9a-f]+}} + // CHECK-NEXT: ReturnStmt 0x{{[0-9a-f]+}} + constexpr bool cm() { [[msvc::constexpr]] return vm(); } +}; +static_assert(S1{}.cm()); diff --git a/clang/test/Analysis/enum-cast-out-of-range.cpp b/clang/test/Analysis/enum-cast-out-of-range.cpp index 0eb740664ecdc7c8009ef0b04a8214036794e7bd..09835d420672bd2b7801c7c329ff5d2e34504d20 100644 --- a/clang/test/Analysis/enum-cast-out-of-range.cpp +++ b/clang/test/Analysis/enum-cast-out-of-range.cpp @@ -43,115 +43,115 @@ struct S { }; void unscopedUnspecified() { - unscoped_unspecified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning {{The value '-5' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} unscoped_unspecified_t ValidNegativeValue1 = static_cast(-4); // OK. unscoped_unspecified_t ValidNegativeValue2 = static_cast(-3); // OK. - unscoped_unspecified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} - unscoped_unspecified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} - unscoped_unspecified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} unscoped_unspecified_t ValidPositiveValue1 = static_cast(1); // OK. unscoped_unspecified_t ValidPositiveValue2 = static_cast(2); // OK. - unscoped_unspecified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} unscoped_unspecified_t ValidPositiveValue3 = static_cast(4); // OK. - unscoped_unspecified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} } void unscopedSpecified() { - unscoped_specified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning {{The value '-5' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} unscoped_specified_t ValidNegativeValue1 = static_cast(-4); // OK. unscoped_specified_t ValidNegativeValue2 = static_cast(-3); // OK. - unscoped_specified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} - unscoped_specified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} - unscoped_specified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} unscoped_specified_t ValidPositiveValue1 = static_cast(1); // OK. unscoped_specified_t ValidPositiveValue2 = static_cast(2); // OK. - unscoped_specified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} unscoped_specified_t ValidPositiveValue3 = static_cast(4); // OK. - unscoped_specified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} } void scopedUnspecified() { - scoped_unspecified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning{{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning{{The value '-5' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} scoped_unspecified_t ValidNegativeValue1 = static_cast(-4); // OK. scoped_unspecified_t ValidNegativeValue2 = static_cast(-3); // OK. - scoped_unspecified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} - scoped_unspecified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} - scoped_unspecified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} scoped_unspecified_t ValidPositiveValue1 = static_cast(1); // OK. scoped_unspecified_t ValidPositiveValue2 = static_cast(2); // OK. - scoped_unspecified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} scoped_unspecified_t ValidPositiveValue3 = static_cast(4); // OK. - scoped_unspecified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} } void scopedSpecified() { - scoped_specified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning {{The value '-5' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} scoped_specified_t ValidNegativeValue1 = static_cast(-4); // OK. scoped_specified_t ValidNegativeValue2 = static_cast(-3); // OK. - scoped_specified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} - scoped_specified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} - scoped_specified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} scoped_specified_t ValidPositiveValue1 = static_cast(1); // OK. scoped_specified_t ValidPositiveValue2 = static_cast(2); // OK. - scoped_specified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} scoped_specified_t ValidPositiveValue3 = static_cast(4); // OK. - scoped_specified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} } void unscopedUnspecifiedCStyle() { - unscoped_unspecified_t InvalidBeforeRangeBegin = (unscoped_unspecified_t)(-5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidBeforeRangeBegin = (unscoped_unspecified_t)(-5); // expected-warning {{The value '-5' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} unscoped_unspecified_t ValidNegativeValue1 = (unscoped_unspecified_t)(-4); // OK. unscoped_unspecified_t ValidNegativeValue2 = (unscoped_unspecified_t)(-3); // OK. - unscoped_unspecified_t InvalidInsideRange1 = (unscoped_unspecified_t)(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} - unscoped_unspecified_t InvalidInsideRange2 = (unscoped_unspecified_t)(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} - unscoped_unspecified_t InvalidInsideRange3 = (unscoped_unspecified_t)(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange1 = (unscoped_unspecified_t)(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange2 = (unscoped_unspecified_t)(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange3 = (unscoped_unspecified_t)(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} unscoped_unspecified_t ValidPositiveValue1 = (unscoped_unspecified_t)(1); // OK. unscoped_unspecified_t ValidPositiveValue2 = (unscoped_unspecified_t)(2); // OK. - unscoped_unspecified_t InvalidInsideRange4 = (unscoped_unspecified_t)(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange4 = (unscoped_unspecified_t)(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} unscoped_unspecified_t ValidPositiveValue3 = (unscoped_unspecified_t)(4); // OK. - unscoped_unspecified_t InvalidAfterRangeEnd = (unscoped_unspecified_t)(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidAfterRangeEnd = (unscoped_unspecified_t)(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} } void unscopedSpecifiedCStyle() { - unscoped_specified_t InvalidBeforeRangeBegin = (unscoped_specified_t)(-5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidBeforeRangeBegin = (unscoped_specified_t)(-5); // expected-warning {{The value '-5' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} unscoped_specified_t ValidNegativeValue1 = (unscoped_specified_t)(-4); // OK. unscoped_specified_t ValidNegativeValue2 = (unscoped_specified_t)(-3); // OK. - unscoped_specified_t InvalidInsideRange1 = (unscoped_specified_t)(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} - unscoped_specified_t InvalidInsideRange2 = (unscoped_specified_t)(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} - unscoped_specified_t InvalidInsideRange3 = (unscoped_specified_t)(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange1 = (unscoped_specified_t)(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange2 = (unscoped_specified_t)(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange3 = (unscoped_specified_t)(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} unscoped_specified_t ValidPositiveValue1 = (unscoped_specified_t)(1); // OK. unscoped_specified_t ValidPositiveValue2 = (unscoped_specified_t)(2); // OK. - unscoped_specified_t InvalidInsideRange4 = (unscoped_specified_t)(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange4 = (unscoped_specified_t)(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} unscoped_specified_t ValidPositiveValue3 = (unscoped_specified_t)(4); // OK. - unscoped_specified_t InvalidAfterRangeEnd = (unscoped_specified_t)(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidAfterRangeEnd = (unscoped_specified_t)(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} } void scopedUnspecifiedCStyle() { - scoped_unspecified_t InvalidBeforeRangeBegin = (scoped_unspecified_t)(-5); // expected-warning{{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidBeforeRangeBegin = (scoped_unspecified_t)(-5); // expected-warning{{The value '-5' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} scoped_unspecified_t ValidNegativeValue1 = (scoped_unspecified_t)(-4); // OK. scoped_unspecified_t ValidNegativeValue2 = (scoped_unspecified_t)(-3); // OK. - scoped_unspecified_t InvalidInsideRange1 = (scoped_unspecified_t)(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} - scoped_unspecified_t InvalidInsideRange2 = (scoped_unspecified_t)(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} - scoped_unspecified_t InvalidInsideRange3 = (scoped_unspecified_t)(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange1 = (scoped_unspecified_t)(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange2 = (scoped_unspecified_t)(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange3 = (scoped_unspecified_t)(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} scoped_unspecified_t ValidPositiveValue1 = (scoped_unspecified_t)(1); // OK. scoped_unspecified_t ValidPositiveValue2 = (scoped_unspecified_t)(2); // OK. - scoped_unspecified_t InvalidInsideRange4 = (scoped_unspecified_t)(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange4 = (scoped_unspecified_t)(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} scoped_unspecified_t ValidPositiveValue3 = (scoped_unspecified_t)(4); // OK. - scoped_unspecified_t InvalidAfterRangeEnd = (scoped_unspecified_t)(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidAfterRangeEnd = (scoped_unspecified_t)(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} } void scopedSpecifiedCStyle() { - scoped_specified_t InvalidBeforeRangeBegin = (scoped_specified_t)(-5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidBeforeRangeBegin = (scoped_specified_t)(-5); // expected-warning {{The value '-5' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} scoped_specified_t ValidNegativeValue1 = (scoped_specified_t)(-4); // OK. scoped_specified_t ValidNegativeValue2 = (scoped_specified_t)(-3); // OK. - scoped_specified_t InvalidInsideRange1 = (scoped_specified_t)(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} - scoped_specified_t InvalidInsideRange2 = (scoped_specified_t)(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} - scoped_specified_t InvalidInsideRange3 = (scoped_specified_t)(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange1 = (scoped_specified_t)(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange2 = (scoped_specified_t)(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange3 = (scoped_specified_t)(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} scoped_specified_t ValidPositiveValue1 = (scoped_specified_t)(1); // OK. scoped_specified_t ValidPositiveValue2 = (scoped_specified_t)(2); // OK. - scoped_specified_t InvalidInsideRange4 = (scoped_specified_t)(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange4 = (scoped_specified_t)(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} scoped_specified_t ValidPositiveValue3 = (scoped_specified_t)(4); // OK. - scoped_specified_t InvalidAfterRangeEnd = (scoped_specified_t)(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidAfterRangeEnd = (scoped_specified_t)(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} } unscoped_unspecified_t unused; @@ -194,13 +194,13 @@ void rangeConstrained6(int input) { void rangeConstrained7(int input) { if (input >= 3 && input <= 3) - auto value = static_cast(input); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + auto value = static_cast(input); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} } void enumBitFieldAssignment() { S s; s.E = static_cast(4); // OK. - s.E = static_cast(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + s.E = static_cast(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} } diff --git a/clang/test/C/C2x/n2836_n2939.c b/clang/test/C/C2x/n2836_n2939.c new file mode 100644 index 0000000000000000000000000000000000000000..13e23c3def80ddc59013d37cd51581ad3c034275 --- /dev/null +++ b/clang/test/C/C2x/n2836_n2939.c @@ -0,0 +1,83 @@ +// RUN: %clang_cc1 -x c -std=c2x -fsyntax-only -verify %s +// RUN: %clang_cc1 -x c -std=c2x -E -DPP_ONLY=1 %s | FileCheck %s --strict-whitespace + +/* WG14 N2836: Clang 15 + * Identifier Syntax using Unicode Standard Annex 31 + */ + +/* WG14 N2939: Clang 15 + * Identifier Syntax Fixes + */ + +// Some of the tests below are derived from clang/test/Lexer/unicode.c. + +// This file contains Unicode characters; please do not "fix" them! + +// No diagnostics for pragma directives. +#pragma mark ¡Unicode! + +// lone non-identifier characters are allowed in preprocessing. +#define COPYRIGHT Copyright © 2012 +#define XSTR(X) #X +#define STR(X) XSTR(X) + +static const char *copyright = STR(COPYRIGHT); // no-warning +// CHECK: static const char *copyright = "Copyright © {{2012}}"; + +#if PP_ONLY +COPYRIGHT +// CHECK: Copyright © {{2012}} +#endif + +// The characters in the following identifiers are no longer valid as either +// start or continuation characters as of C23. These are taken from section 1 +// of N2836. +extern int \N{CONSTRUCTION WORKER}; // expected-error {{expected identifier or '('}} +extern int X\N{CONSTRUCTION WORKER}; // expected-error {{character not allowed in an identifier}} +extern int \U0001F477; // expected-error {{expected identifier or '('}} +extern int X\U0001F477; // expected-error {{character not allowed in an identifier}} +extern int 👷; // expected-error {{unexpected character }} \ + // expected-warning {{declaration does not declare anything}} +extern int X👷; // expected-error {{character not allowed in an identifier}} +extern int 🕐; // expected-error {{unexpected character }} \ + // expected-warning {{declaration does not declare anything}} +extern int X🕐; // expected-error {{character not allowed in an identifier}} +extern int 💀; // expected-error {{unexpected character }} \ + // expected-warning {{declaration does not declare anything}} +extern int X💀; // expected-error {{character not allowed in an identifier}} +extern int 👊; // expected-error {{unexpected character }} \ + // expected-warning {{declaration does not declare anything}} +extern int X👊; // expected-error {{character not allowed in an identifier}} +extern int 🚀; // expected-error {{unexpected character }} \ + // expected-warning {{declaration does not declare anything}} +extern int X🚀; // expected-error {{character not allowed in an identifier}} +extern int 😀; // expected-error {{unexpected character }} \ + // expected-warning {{declaration does not declare anything}} +extern int X😀; // expected-error {{character not allowed in an identifier}} + +// The characters in the following identifiers are not allowed as start +// characters, but are allowed as continuation characters. +extern int \N{ARABIC-INDIC DIGIT ZERO}; // expected-error {{expected identifier or '('}} +extern int X\N{ARABIC-INDIC DIGIT ZERO}; +extern int \u0661; // expected-error {{expected identifier or '('}} +extern int X\u0661; +extern int ٢; // expected-error {{character not allowed at the start of an identifier}} \\ + // expected-warning {{declaration does not declare anything}} +extern int X٠; + +// The characters in the following identifiers are not valid start or +// continuation characters in the standard, but are accepted as a conforming +// extension. +extern int \N{SUPERSCRIPT ZERO}; // expected-error {{expected identifier or '('}} +extern int X\N{SUPERSCRIPT ZERO}; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int \u00B9; // expected-error {{expected identifier or '('}} +extern int X\u00B9; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int ²; // expected-error {{character not allowed at the start of an identifier}} \\ + // expected-warning {{declaration does not declare anything}} +extern int X²; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int \N{PARTIAL DIFFERENTIAL}; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int X\N{PARTIAL DIFFERENTIAL}; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int \u2207; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int X\u2207; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int ∞; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int X∞; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} diff --git a/clang/test/CXX/class.access/class.friend/p6.cpp b/clang/test/CXX/class.access/class.friend/p6.cpp index 2fe20fe77fc8f2179e6f430a2c0233a7a0ebf9b8..47104e29dc6b3c7554bf750840d5a8da47ad533b 100644 --- a/clang/test/CXX/class.access/class.friend/p6.cpp +++ b/clang/test/CXX/class.access/class.friend/p6.cpp @@ -22,3 +22,16 @@ void local() { friend void f() { } // expected-error{{friend function cannot be defined in a local class}} }; } + +template void f3(T); + +namespace N { + template void f4(T); +} + +template struct A { + friend void f3(T) {} + friend void f3(T) {} // expected-error{{friend function specialization cannot be defined}} + friend void N::f4(T) {} // expected-error{{friend function definition cannot be qualified with 'N::'}} + friend void N::f4(T) {} // expected-error{{friend function definition cannot be qualified with 'N::'}} +}; diff --git a/clang/test/CXX/drs/dr10xx.cpp b/clang/test/CXX/drs/dr10xx.cpp index f30ed1cb3e49673be0d1db66130620ccc459c4d3..77c59078414c69bdb012207d68ba774cf41f64e2 100644 --- a/clang/test/CXX/drs/dr10xx.cpp +++ b/clang/test/CXX/drs/dr10xx.cpp @@ -1,9 +1,9 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors namespace std { __extension__ typedef __SIZE_TYPE__ size_t; @@ -18,30 +18,40 @@ namespace dr1004 { // dr1004: 5 template struct A {}; template struct B1 {}; template class> struct B2 {}; - template void f(); // expected-note {{[with X = dr1004::A]}} - template class X> void f(); // expected-note {{[with X = dr1004::A]}} - template class X> void g(); // expected-note {{[with X = dr1004::A]}} - template void g(); // expected-note {{[with X = dr1004::A]}} + template void f(); // #dr1004-f-1 + template class X> void f(); // #dr1004-f-2 + template class X> void g(); // #dr1004-g-1 + template void g(); // #dr1004-g-2 struct C : A { B1 b1a; B2 b2a; void h() { - f(); // expected-error {{ambiguous}} - g(); // expected-error {{ambiguous}} + f(); + // expected-error@-1 {{call to 'f' is ambiguous}} + // expected-note@#dr1004-f-1 {{candidate function [with X = dr1004::A]}} + // expected-note@#dr1004-f-2 {{candidate function [with X = dr1004::A]}} + g(); + // expected-error@-1 {{call to 'g' is ambiguous}} + // expected-note@#dr1004-g-1 {{candidate function [with X = dr1004::A]}} + // expected-note@#dr1004-g-2 {{candidate function [with X = dr1004::A]}} } }; // This example (from the standard) is actually ill-formed, because // name lookup of "T::template A" names the constructor. - template class U = T::template A> struct Third { }; // expected-error {{is a constructor name}} - Third > t; // expected-note {{in instantiation of default argument}} + template class U = T::template A> struct Third { }; + // expected-error@-1 {{is a constructor name}} + // expected-note@#dr1004-t {{in instantiation of default argument}} + Third > t; // #dr1004-t } namespace dr1042 { // dr1042: 3.5 #if __cplusplus >= 201402L // C++14 added an attribute that we can test the semantics of. - using foo [[deprecated]] = int; // expected-note {{'foo' has been explicitly marked deprecated here}} - foo f = 12; // expected-warning {{'foo' is deprecated}} + using foo [[deprecated]] = int; // #dr1042-using + foo f = 12; + // since-cxx14-warning@-1 {{'foo' is deprecated}} + // since-cxx14-note@#dr1042-using {{'foo' has been explicitly marked deprecated here}} #elif __cplusplus >= 201103L // C++11 did not have any attributes that could be applied to an alias // declaration, so the best we can test is that we accept an empty attribute @@ -76,7 +86,8 @@ namespace dr1054 { // dr1054: no // which copy-initializes a temporary from 'a'. Therefore this is // ill-formed because A does not have a volatile copy constructor. // (We might want to track this aspect under dr1383 instead?) - a; // expected-warning {{assign into a variable to force a volatile load}} + a; + // expected-warning@-1 {{expression result unused; assign into a variable to force a volatile load}} } } diff --git a/clang/test/CXX/drs/dr11xx.cpp b/clang/test/CXX/drs/dr11xx.cpp index 23756ff1927eccf3378142a4c850ec1caf835fad..86e726ae8c7419208ab05efe38df163a8571c14e 100644 --- a/clang/test/CXX/drs/dr11xx.cpp +++ b/clang/test/CXX/drs/dr11xx.cpp @@ -1,30 +1,30 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2a %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2a %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors -namespace dr1111 { // dr1111: yes +namespace dr1111 { // dr1111: 3.2 namespace example1 { -template struct set; +template struct set; // #dr1111-struct-set struct X { - template void set(const T &value); + template void set(const T &value); // #dr1111-func-set }; void foo() { X x; -#pragma clang diagnostic push -#if __cplusplus < 201103L -#pragma clang diagnostic ignored "-Wambiguous-member-template" -#endif + // FIXME: should we backport C++11 behavior? x.set(3.2); -#pragma clang diagnostic pop + // cxx98-error@-1 {{lookup of 'set' in member access expression is ambiguous; using member of 'X'}} + // cxx98-note@#dr1111-func-set {{lookup in the object type 'X' refers here}} + // cxx98-note@#dr1111-struct-set {{lookup from the current scope refers here}} } struct Y {}; void bar() { Y y; - y.set(3.2); // expected-error {{no member named 'set' in 'dr1111::example1::Y'}} + y.set(3.2); + // expected-error@-1 {{no member named 'set' in 'dr1111::example1::Y'}} } } // namespace example1 @@ -46,8 +46,10 @@ void baz() { namespace dr1113 { // dr1113: partial namespace named { - extern int a; // expected-note {{previous}} - static int a; // expected-error {{static declaration of 'a' follows non-static}} + extern int a; // #dr1113-a + static int a; + // expected-error@-1 {{static declaration of 'a' follows non-static}} + // expected-note@#dr1113-a {{previous declaration is here}} } namespace { extern int a; diff --git a/clang/test/CXX/drs/dr12xx.cpp b/clang/test/CXX/drs/dr12xx.cpp index c23a515ba56cb99d80234b1b8bc44e14c32c33a3..adf7f56711c4519b6c8476614dc02a5e9f98dce5 100644 --- a/clang/test/CXX/drs/dr12xx.cpp +++ b/clang/test/CXX/drs/dr12xx.cpp @@ -1,9 +1,9 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected,cxx98-14,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx98-14,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx17,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx17,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx17,since-cxx14,since-cxx11,since-cxx23 -fexceptions -fcxx-exceptions -pedantic-errors // dr1200: na @@ -40,28 +40,33 @@ struct S { S* operator()(); int N; int M; -#if __cplusplus > 202002L +#if __cplusplus >= 202302L template static constexpr auto V = 0; void f(char); void f(int); void mem(S s) { - auto(s)()->M; //expected-warning {{expression result unused}} - auto(s)()->V; //expected-warning {{expression result unused}} + auto(s)()->M; + // since-cxx23-warning@-1 {{expression result unused}} + auto(s)()->V; + // since-cxx23-warning@-1 {{expression result unused}} auto(s)()->f(0); } #endif }; void f(S s) { { -#if __cplusplus > 202002L - auto(s)()->N; //expected-warning {{expression result unused}} +#if __cplusplus >= 202302L + auto(s)()->N; + //since-cxx23-warning@-1 {{expression result unused}} #endif auto(s)()->M; } { - S(s)()->N; //expected-warning {{expression result unused}} - S(s)()->M; //expected-warning {{expression result unused}} + S(s)()->N; + // since-cxx11-warning@-1 {{expression result unused}} + S(s)()->M; + // since-cxx11-warning@-1 {{expression result unused}} } } @@ -74,23 +79,27 @@ void g() { A a(B ()->C); A b(auto ()->C); static_assert(sizeof(B ()->C[1] == sizeof(int)), ""); - sizeof(auto () -> C[1]); // expected-error{{function cannot return array type 'C[1]'}} + sizeof(auto () -> C[1]); + // since-cxx11-error@-1 {{function cannot return array type 'C[1]' (aka 'dr1223::BB[1]')}} } } #endif #if __cplusplus >= 201103L -namespace dr1227 { // dr1227: yes -template struct A { using X = typename T::X; }; // expected-error {{type 'int' cannot be used prior to '::' because it has no members}} +namespace dr1227 { // dr1227: 3.0 +template struct A { using X = typename T::X; }; +// since-cxx11-error@-1 {{type 'int' cannot be used prior to '::' because it has no members}} +// since-cxx11-note@#dr1227-g {{in instantiation of template class 'dr1227::A' requested here}} +// since-cxx11-note@#dr1227-g-int {{while substituting explicitly-specified template arguments into function template 'g'}} template typename T::X f(typename A::X); template void f(...) { } -template auto g(typename A::X) -> typename T::X; // expected-note {{in instantiation of template class 'dr1227::A' requested here}} +template auto g(typename A::X) -> typename T::X; // #dr1227-g template void g(...) { } void h() { f(0); // OK, substituting return type causes deduction to fail - g(0); // expected-note {{while substituting explicitly-specified template arguments into function template 'g'}} + g(0); // #dr1227-g-int } } #endif @@ -109,15 +118,21 @@ struct Derived : Base { namespace dr1265 { // dr1265: 5 #if __cplusplus >= 201103L - auto a = 0, b() -> int; // expected-error {{declaration with trailing return type must be the only declaration in its group}} - auto b() -> int, d = 0; // expected-error {{declaration with trailing return type must be the only declaration in its group}} - auto e() -> int, f() -> int; // expected-error {{declaration with trailing return type must be the only declaration in its group}} + auto a = 0, b() -> int; + // since-cxx11-error@-1 {{declaration with trailing return type must be the only declaration in its group}} + auto b() -> int, d = 0; + // since-cxx11-error@-1 {{declaration with trailing return type must be the only declaration in its group}} + auto e() -> int, f() -> int; + // since-cxx11-error@-1 {{declaration with trailing return type must be the only declaration in its group}} #endif #if __cplusplus >= 201402L - auto g(), h = 0; // expected-error {{function with deduced return type must be the only declaration in its group}} - auto i = 0, j(); // expected-error {{function with deduced return type must be the only declaration in its group}} - auto k(), l(); // expected-error {{function with deduced return type must be the only declaration in its group}} + auto g(), h = 0; + // since-cxx14-error@-1 {{function with deduced return type must be the only declaration in its group}} + auto i = 0, j(); + // since-cxx14-error@-1 {{function with deduced return type must be the only declaration in its group}} + auto k(), l(); + // since-cxx14-error@-1 {{function with deduced return type must be the only declaration in its group}} #endif } @@ -130,16 +145,16 @@ namespace dr1295 { // dr1295: 4 X x = {1}; - unsigned const &r1 = static_cast(x).bitfield; // expected-error 0-1{{C++11}} - unsigned const &r2 = static_cast(x.bitfield); // expected-error 0-1{{C++11}} + unsigned const &r1 = static_cast(x).bitfield; + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + unsigned const &r2 = static_cast(x.bitfield); + // cxx98-error@-1 {{rvalue references are a C++11 extension}} - template struct Y {}; - Y y; -#if __cplusplus <= 201402L - // expected-error@-2 {{does not refer to any declaration}} expected-note@-3 {{here}} -#else - // expected-error@-4 {{refers to subobject}} -#endif + template struct Y {}; // #dr1295-Y + Y y; // #dr1295-y + // cxx98-14-error@-1 {{non-type template argument does not refer to any declaration}} + // cxx98-14-note@#dr1295-Y {{template parameter is declared here}} + // since-cxx17-error@#dr1295-y {{non-type template argument refers to subobject 'x.bitfield'}} #if __cplusplus >= 201103L const unsigned other = 0; diff --git a/clang/test/CXX/drs/dr13xx.cpp b/clang/test/CXX/drs/dr13xx.cpp index 3510695954e27c11335fc6d42db9ef39a579354d..359c04b3e0f3d4e011cf55c05c092ac113a11442 100644 --- a/clang/test/CXX/drs/dr13xx.cpp +++ b/clang/test/CXX/drs/dr13xx.cpp @@ -1,7 +1,10 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected,cxx98-14,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx11-17,cxx11-14,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx11-17,cxx11-14,since-cxx14,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,cxx11-17,since-cxx14,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors __extension__ typedef __SIZE_TYPE__ size_t; @@ -14,11 +17,13 @@ namespace std { } #if __cplusplus >= 201103L -namespace dr1305 { // dr1305: yes -struct Incomplete; // expected-note {{forward declaration of 'dr1305::Incomplete'}} +namespace dr1305 { // dr1305: 3.0 +struct Incomplete; // #dr1305-Incomplete struct Complete {}; -int incomplete = alignof(Incomplete(&)[]); // expected-error {{invalid application of 'alignof' to an incomplete type 'Incomplete'}} +int incomplete = alignof(Incomplete(&)[]); +// since-cxx11-error@-1 {{invalid application of 'alignof' to an incomplete type 'Incomplete'}} +// since-cxx11-note@#dr1305-Incomplete {{forward declaration of 'dr1305::Incomplete'}} int complete = alignof(Complete(&)[]); } #endif @@ -36,9 +41,11 @@ void caller() { } // namespace dr1307 namespace dr1310 { // dr1310: 5 - struct S {} * sp = new S::S; // expected-error {{qualified reference to 'S' is a constructor name}} + struct S {} * sp = new S::S; + // expected-error@-1 {{qualified reference to 'S' is a constructor name rather than a type in this context}} void f() { - S::S(a); // expected-error {{qualified reference to 'S' is a constructor name}} + S::S(a); + // expected-error@-1 {{qualified reference to 'S' is a constructor name rather than a type in this context}} } struct T { int n; typedef int U; typedef T V; }; int k = T().T::T::n; @@ -64,39 +71,67 @@ namespace dr1310 { // dr1310: 5 template struct W : WBase { typedef int X; int n; }; void w_test() { - W::W w1a; // expected-error {{qualified reference to 'W' is a constructor name}} + W::W w1a; + // expected-error@-1 {{qualified reference to 'W' is a constructor name rather than a type in this context}} W::W::X w1ax; - W::W w1b; // expected-error {{qualified reference to 'W' is a constructor name}} + W::W w1b; + // expected-error@-1 {{qualified reference to 'W' is a constructor name rather than a template name in this context}} W::W::X w1bx; - typename W::W w2a; // expected-error {{qualified reference to 'W' is a constructor name}} expected-error 0-1{{outside of a template}} - typename W::W::X w2ax; // expected-error 0-1{{outside of a template}} - typename W::W w2b; // expected-error {{qualified reference to 'W' is a constructor name}} expected-error 0-1{{outside of a template}} - typename W::W::X w2bx; // expected-error 0-1{{outside of a template}} - W::template W w3; // expected-error {{qualified reference to 'W' is a constructor name}} expected-error 0-1{{outside of a template}} - W::template W::X w3x; // expected-error 0-1{{outside of a template}} - typename W::template W w4; // expected-error {{qualified reference to 'W' is a constructor name}} expected-error 0-2{{outside of a template}} - typename W::template W::X w4x; // expected-error 0-2{{outside of a template}} - - TT::W> tt1; // expected-error {{qualified reference to 'W' is a constructor name}} - TTy::W> tt1a; // expected-error {{qualified reference to 'W' is a constructor name}} - TT::template W> tt2; // expected-error {{qualified reference to 'W' is a constructor name}} expected-error 0-1{{outside of a template}} + typename W::W w2a; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a type in this context, despite preceding 'typename' keyword}} + // cxx98-error@-2 {{'typename' occurs outside of a template}} + typename W::W::X w2ax; + // cxx98-error@-1 {{'typename' occurs outside of a template}} + typename W::W w2b; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a template name in this context, despite preceding 'typename' keyword}} + // cxx98-error@-2 {{'typename' occurs outside of a template}} + typename W::W::X w2bx; + // cxx98-error@-1 {{'typename' occurs outside of a template}} + W::template W w3; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a template name in this context, despite preceding 'template' keyword}} + // cxx98-error@-2 {{'template' keyword outside of a template}} + W::template W::X w3x; + // cxx98-error@-1 {{'template' keyword outside of a template}} + typename W::template W w4; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a template name in this context, despite preceding 'template' keyword}} + // cxx98-error@-2 {{'template' keyword outside of a template}} + // cxx98-error@-3 {{'typename' occurs outside of a template}} + typename W::template W::X w4x; + // cxx98-error@-1 {{'template' keyword outside of a template}} + // cxx98-error@-2 {{'typename' occurs outside of a template}} + + TT::W> tt1; + // expected-error@-1 {{qualified reference to 'W' is a constructor name rather than a type in this context}} + TTy::W> tt1a; + // expected-error@-1 {{qualified reference to 'W' is a constructor name rather than a type in this context}} + TT::template W> tt2; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a template name in this context, despite preceding 'template' keyword}} + // cxx98-error@-2 {{'template' keyword outside of a template}} TT::WBase> tt3; TTy::WBase> tt3a; - TT::template WBase> tt4; // expected-error 0-1{{outside of a template}} + TT::template WBase> tt4; + // cxx98-error@-1 {{'template' keyword outside of a template}} W w; (void)w.W::W::n; (void)w.W::W::n; (void)w.W::W::n; - (void)w.W::template W::n; // expected-error 0-1{{outside of a template}} + (void)w.W::template W::n; + // cxx98-error@-1 {{'template' keyword outside of a template}} } template void wt_test() { - typename W::W w2a; // expected-error {{qualified reference to 'W' is a constructor name}} - typename W::template W w4; // expected-error {{qualified reference to 'W' is a constructor name}} - TTy tt2; // expected-error {{qualified reference to 'W' is a constructor name}} - TT tt3; // expected-error {{qualified reference to 'W' is a constructor name}} + typename W::W w2a; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a type in this context, despite preceding 'typename' keyword}} + // cxx98-note@#dr1310-W-int {{in instantiation of function template specialization 'dr1310::wt_test >' requested here}} + // since-cxx11-note@#dr1310-W-int {{in instantiation of function template specialization 'dr1310::wt_test>' requested here}} + typename W::template W w4; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a template name in this context, despite preceding 'template' keyword}} + TTy tt2; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a type in this context, despite preceding 'typename' keyword}} + TT tt3; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a template name in this context, despite preceding 'template' keyword}} } template void wt_test_good() { @@ -111,18 +146,19 @@ namespace dr1310 { // dr1310: 5 (void)w.template W::W::n; (void)w.template W::template W::n; } - template void wt_test >(); // expected-note {{instantiation of}} + template void wt_test >(); // #dr1310-W-int template void wt_test_good >(); } namespace dr1315 { // dr1315: partial template struct A {}; - template // expected-note {{non-deducible template parameter 'I'}} - struct A {}; // expected-error {{contains a template parameter that cannot be deduced}} + template struct A {}; + // expected-error@-1 {{class template partial specialization contains a template parameter that cannot be deduced; this partial specialization will never be used}} + // expected-note@-2 {{non-deducible template parameter 'I'}} template struct A {}; template struct B; - template struct B {}; // expected-note {{matches}} + template struct B {}; // #dr1315-B-1 B<1, 2, 3> b1; // Multiple declarations with the same dependent expression are equivalent @@ -131,8 +167,11 @@ namespace dr1315 { // dr1315: partial B<1, 2, 2>::type b2; // Multiple declarations with differing dependent expressions are unordered. - template struct B {}; // expected-note {{matches}} - B<1, 2, 4> b3; // expected-error {{ambiguous}} + template struct B {}; // #dr1315-B-2 + B<1, 2, 4> b3; + // expected-error@-1 {{ambiguous partial specializations of 'B<1, 2, 4>'}} + // expected-note@#dr1315-B-1 {{partial specialization matches [with I = 1, K = 4]}} + // expected-note@#dr1315-B-2 {{partial specialization matches [with I = 1, K = 4]}} // FIXME: Under dr1315, this is perhaps valid, but that is not clear: this // fails the "more specialized than the primary template" test because the @@ -146,7 +185,9 @@ namespace dr1315 { // dr1315: partial namespace dr1330 { // dr1330: 4 c++11 // exception-specifications are parsed in a context where the class is complete. struct A { - void f() throw(T) {} // expected-error 0-1{{C++17}} expected-note 0-1{{noexcept}} + void f() throw(T) {} + // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}} + // since-cxx17-note@-2 {{use 'noexcept(false)' instead}} struct T {}; #if __cplusplus >= 201103L @@ -156,8 +197,12 @@ namespace dr1330 { // dr1330: 4 c++11 #endif }; - void (A::*af1)() throw(A::T) = &A::f; // expected-error 0-1{{C++17}} expected-note 0-1{{noexcept}} - void (A::*af2)() throw() = &A::f; // expected-error-re {{{{not superset|different exception spec}}}} + void (A::*af1)() throw(A::T) = &A::f; + // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}} + // since-cxx17-note@-2 {{use 'noexcept(false)' instead}} + void (A::*af2)() throw() = &A::f; + // cxx98-14-error@-1 {{target exception specification is not superset of source}} + // since-cxx17-error@-2 {{cannot initialize a variable of type 'void (dr1330::A::*)() throw()' with an rvalue of type 'void (dr1330::A::*)() throw(T)': different exception specifications}} #if __cplusplus >= 201103L static_assert(noexcept(A().g()), ""); @@ -166,7 +211,9 @@ namespace dr1330 { // dr1330: 4 c++11 // Likewise, they're instantiated separately from an enclosing class template. template struct B { - void f() throw(T, typename U::type) {} // expected-error 0-1{{C++17}} expected-note 0-1{{noexcept}} + void f() throw(T, typename U::type) {} + // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}} + // since-cxx17-note@-2 {{use 'noexcept(false)' instead}} struct T {}; #if __cplusplus >= 201103L @@ -183,9 +230,6 @@ namespace dr1330 { // dr1330: 4 c++11 static const int value = true; }; - void (B

::*bpf1)() throw(B

::T, int) = &B

::f; // expected-error 0-1{{C++17}} expected-note 0-1{{noexcept}} -#if __cplusplus < 201103L - // expected-error@-2 {{not superset}} // FIXME: We only delay instantiation in C++11 onwards. In C++98, something // weird happens: instantiation of B

fails because it references T before // it's instantiated, but the diagnostic is suppressed in @@ -193,20 +237,20 @@ namespace dr1330 { // dr1330: 4 c++11 // obviously a bad way to react to this situation; we should still producing // the "T has not yet been instantiated" error here, rather than giving // confusing errors later on. -#endif - void (B

::*bpf2)() throw(int) = &B

::f; // expected-error 0-1{{C++17}} expected-note 0-1{{noexcept}} -#if __cplusplus <= 201402L - // expected-error@-2 {{not superset}} -#else - // expected-warning@-4 {{not superset}} -#endif + void (B

::*bpf1)() throw(B

::T, int) = &B

::f; + // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}} + // since-cxx17-note@-2 {{use 'noexcept(false)' instead}} + // cxx98-error@-3 {{target exception specification is not superset of source}} + + void (B

::*bpf2)() throw(int) = &B

::f; + // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}} + // since-cxx17-note@-2 {{use 'noexcept(false)' instead}} + // cxx98-14-error@-3 {{target exception specification is not superset of source}} + // since-cxx17-warning@-4 {{target exception specification is not superset of source}} void (B

::*bpf3)() = &B

::f; void (B

::*bpf4)() throw() = &B

::f; -#if __cplusplus <= 201402L - // expected-error@-2 {{not superset}} -#else - // expected-error@-4 {{different exception specifications}} -#endif + // cxx98-14-error@-1 {{target exception specification is not superset of source}} + // since-cxx17-error@-2 {{cannot initialize a variable of type 'void (B

::*)() throw()' with an rvalue of type 'void (dr1330::B::*)() throw(T, typename P::type)': different exception specifications}} #if __cplusplus >= 201103L static_assert(noexcept(B

{ -#if __cplusplus < 201103L +#if __cplusplus == 199711L static const int n = 0; #else static const int n = *P; @@ -40,11 +40,13 @@ namespace dr2100 { // dr2100: 12 template struct B { static const int n = 1; int f() { - return Y::declared_later; // expected-error {{no member named 'declared_later'}} + return Y::declared_later; + // expected-error@-1 {{no member named 'declared_later' in 'dr2100::Y<1>'}} } int g() { static const int n = 2; - return Y::declared_later; // expected-error {{no member named 'declared_later'}} + return Y::declared_later; + // expected-error@-1 {{no member named 'declared_later' in 'dr2100::Y<2>'}} } }; template struct Y { @@ -55,10 +57,12 @@ namespace dr2100 { // dr2100: 12 namespace dr2103 { // dr2103: yes void f() { int a; - int &r = a; // expected-note {{here}} + int &r = a; // #dr2103-r struct Inner { void f() { - int &s = r; // expected-error {{enclosing function}} + int &s = r; + // expected-error@-1 {{reference to local variable 'r' declared in enclosing function 'dr2103::f'}} + // expected-note@#dr2103-r {{'r' declared here}} (void)s; } }; @@ -84,28 +88,46 @@ namespace dr2126 { // dr2126: 12 A &b = (A &)(const A &)A{1}; // const temporary A &&c = (A &&)(const A &)A{1}; // const temporary - A &&d = {1}; // non-const temporary expected-note {{here}} - const A &e = (A &)(A &&) A{1}; // non-const temporary expected-note {{here}} - A &&f = (A &&)(A &&) A{1}; // non-const temporary expected-note {{here}} + A &&d = {1}; // non-const temporary #dr21260-d + const A &e = (A &)(A &&) A{1}; // non-const temporary #dr21260-e + A &&f = (A &&)(A &&) A{1}; // non-const temporary #dr21260-f constexpr const A &g = {1}; // const temporary - constexpr A &&h = {1}; // non-const temporary expected-note {{here}} + constexpr A &&h = {1}; // non-const temporary #dr21260-h struct B { const A &a; }; - B i = {{1}}; // extending decl not usable in constant expr expected-note {{here}} - const B j = {{1}}; // extending decl not usable in constant expr expected-note {{here}} + B i = {{1}}; // extending decl not usable in constant expr #dr21260-i + const B j = {{1}}; // extending decl not usable in constant expr #dr21260-j constexpr B k = {{1}}; // extending decl usable in constant expr static_assert(a.n == 1, ""); static_assert(b.n == 1, ""); static_assert(c.n == 1, ""); - static_assert(d.n == 1, ""); // expected-error {{constant}} expected-note {{read of temporary}} - static_assert(e.n == 1, ""); // expected-error {{constant}} expected-note {{read of temporary}} - static_assert(f.n == 1, ""); // expected-error {{constant}} expected-note {{read of temporary}} + static_assert(d.n == 1, ""); + // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}} + // since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}} + // since-cxx11-note@#dr21260-d {{temporary created here}} + static_assert(e.n == 1, ""); + // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}} + // since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}} + // since-cxx11-note@#dr21260-e {{temporary created here}} + static_assert(f.n == 1, ""); + // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}} + // since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}} + // since-cxx11-note@#dr21260-f {{temporary created here}} static_assert(g.n == 1, ""); - static_assert(h.n == 1, ""); // expected-error {{constant}} expected-note {{read of temporary}} - static_assert(i.a.n == 1, ""); // expected-error {{constant}} expected-note {{read of non-constexpr variable}} - static_assert(j.a.n == 1, ""); // expected-error {{constant}} expected-note {{read of temporary}} + static_assert(h.n == 1, ""); + // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}} + // since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}} + // since-cxx11-note@#dr21260-h {{temporary created here}} + static_assert(i.a.n == 1, ""); + // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}} + // since-cxx11-note@-2 {{read of non-constexpr variable 'i' is not allowed in a constant expression}} + // since-cxx11-note@#dr21260-i {{declared here}} + static_assert(j.a.n == 1, ""); + // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}} + // since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}} + // since-cxx11-note@#dr21260-j {{temporary created here}} static_assert(k.a.n == 1, ""); #endif } @@ -128,19 +150,27 @@ struct B{}; void foo() { struct A *b = (1 == 1) ? new struct A : new struct A; - struct S *a = (1 == 1) ? new struct S : new struct S; // expected-error 2{{allocation of incomplete type}} // expected-note 2{{forward}} + struct S *a = (1 == 1) ? new struct S : new struct S; + // expected-error@-1 {{allocation of incomplete type 'struct S'}} + // expected-note@-2 {{forward declaration of 'S'}} + // expected-error@-3 {{allocation of incomplete type 'struct S'}} + // expected-note@-4 {{forward declaration of 'S'}} #if __cplusplus >= 201103L A *aa = new struct A{}; B *bb = new struct B{}; - (void)new struct C{}; // expected-error {{allocation of incomplete type }} // expected-note {{forward}} + (void)new struct C{}; + // since-cxx11-error@-1 {{allocation of incomplete type 'struct C'}} + // since-cxx11-note@-2 {{forward declaration of 'C'}} struct A *c = (1 == 1) ? new struct A {} : new struct A {}; - alignof(struct D{}); // expected-error {{cannot be defined in a type specifier}} + alignof(struct D{}); + // since-cxx11-error@-1 {{'D' cannot be defined in a type specifier}} #endif - sizeof(struct E{}); // expected-error {{cannot be defined in a type specifier}} + sizeof(struct E{}); + // expected-error@-1 {{'E' cannot be defined in a type specifier}} } } @@ -149,7 +179,8 @@ namespace dr2157 { // dr2157: 11 #if __cplusplus >= 201103L enum E : int; struct X { - enum dr2157::E : int(); // expected-error {{only allows ':' in member enumeration declaration to introduce a fixed underlying type}} + enum dr2157::E : int(); + // since-cxx11-error@-1 {{ISO C++ only allows ':' in member enumeration declaration to introduce a fixed underlying type, not an anonymous bit-field}} }; #endif } @@ -159,11 +190,13 @@ namespace dr2157 { // dr2157: 11 namespace dr2170 { // dr2170: 9 #if __cplusplus >= 201103L void f() { - constexpr int arr[3] = {1, 2, 3}; // expected-note {{here}} + constexpr int arr[3] = {1, 2, 3}; // #dr2170-arr struct S { int get(int n) { return arr[n]; } - const int &get_ref(int n) { return arr[n]; } // expected-error {{enclosing function}} - // FIXME: expected-warning@-1 {{reference to stack}} + const int &get_ref(int n) { return arr[n]; } + // since-cxx11-warning@-1 {{reference to stack memory associated with local variable 'arr' returned}} FIXME + // since-cxx11-error@-2 {{reference to local variable 'arr' declared in enclosing function 'dr2170::f'}} + // since-cxx11-note@#dr2170-arr {{'arr' declared here}} }; } #endif @@ -198,22 +231,32 @@ static_assert(!__is_trivially_assignable(NonConstCopy &&, NonConstCopy &&), ""); namespace dr2180 { // dr2180: yes class A { - A &operator=(const A &); // expected-note 0-2{{here}} - A &operator=(A &&); // expected-note 0-2{{here}} expected-error 0-1{{extension}} + A &operator=(const A &); // #dr2180-A-copy + A &operator=(A &&); // #dr2180-A-move + // cxx98-error@-1 {{rvalue references are a C++11 extension}} }; - struct B : virtual A { + struct B : virtual A { // #dr2180-B B &operator=(const B &); - B &operator=(B &&); // expected-error 0-1{{extension}} + B &operator=(B &&); + // cxx98-error@-1 {{rvalue references are a C++11 extension}} virtual void foo() = 0; }; -#if __cplusplus < 201103L - B &B::operator=(const B&) = default; // expected-error {{private member}} expected-error {{extension}} expected-note {{here}} - B &B::operator=(B&&) = default; // expected-error {{private member}} expected-error 2{{extension}} expected-note {{here}} -#else - B &B::operator=(const B&) = default; // expected-error {{would delete}} expected-note@-9{{inaccessible copy assignment}} - B &B::operator=(B&&) = default; // expected-error {{would delete}} expected-note@-10{{inaccessible move assignment}} -#endif + B &B::operator=(const B&) = default; // #dr2180-B-copy + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-2 {{'operator=' is a private member of 'dr2180::A'}} + // cxx98-note@-3 {{in defaulted copy assignment operator for 'dr2180::B' first required here}} + // cxx98-note@#dr2180-A-copy {{implicitly declared private here}} + // since-cxx11-error@#dr2180-B-copy {{defaulting this copy assignment operator would delete it after its first declaration}} + // since-cxx11-note@#dr2180-B {{copy assignment operator of 'B' is implicitly deleted because base class 'A' has an inaccessible copy assignment operator}} + B &B::operator=(B&&) = default; // #dr2180-B-move + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + // cxx98-error@-2 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-3 {{'operator=' is a private member of 'dr2180::A'}} + // cxx98-note@-4 {{in defaulted move assignment operator for 'dr2180::B' first required here}} + // cxx98-note@#dr2180-A-move {{implicitly declared private here}} + // since-cxx11-error@#dr2180-B-move {{defaulting this move assignment operator would delete it after its first declaration}} + // since-cxx11-note@#dr2180-B {{move assignment operator of 'B' is implicitly deleted because base class 'A' has an inaccessible move assignment operator}} } namespace dr2199 { // dr2199: 3.8 diff --git a/clang/test/CXX/drs/dr22xx.cpp b/clang/test/CXX/drs/dr22xx.cpp index cd849443b1119bac90c7e62a736584ddb923e55e..19518247b5289c3921a320eab4aa5f5065bd3477 100644 --- a/clang/test/CXX/drs/dr22xx.cpp +++ b/clang/test/CXX/drs/dr22xx.cpp @@ -1,14 +1,19 @@ -// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++1z -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors + #if __cplusplus >= 201103L namespace dr2211 { // dr2211: 8 void f() { int a; - auto f = [a](int a) { (void)a; }; // expected-error {{a lambda parameter cannot shadow an explicitly captured entity}} - // expected-note@-1{{variable 'a' is explicitly captured here}} + auto f = [a](int a) { (void)a; }; + // since-cxx11-error@-1 {{a lambda parameter cannot shadow an explicitly captured entity}} + // since-cxx11-note@-2 {{variable 'a' is explicitly captured here}} auto g = [=](int a) { (void)a; }; } } @@ -24,9 +29,12 @@ struct A; namespace dr2229 { // dr2229: 7 struct AnonBitfieldQualifiers { - const unsigned : 1; // expected-error {{anonymous bit-field cannot have qualifiers}} - volatile unsigned : 1; // expected-error {{anonymous bit-field cannot have qualifiers}} - const volatile unsigned : 1; // expected-error {{anonymous bit-field cannot have qualifiers}} + const unsigned : 1; + // expected-error@-1 {{anonymous bit-field cannot have qualifiers}} + volatile unsigned : 1; + // expected-error@-1 {{anonymous bit-field cannot have qualifiers}} + const volatile unsigned : 1; + // expected-error@-1 {{anonymous bit-field cannot have qualifiers}} unsigned : 1; const unsigned i1 : 1; @@ -98,7 +106,8 @@ namespace MultilevelSpecialization { template void f(int i = 0, int (&... arr)[V]); }; template<> template - void B::f(int i, int (&arr1)[a], int (&arr2)[b]) {} // expected-error {{does not match}} + void B::f(int i, int (&arr1)[a], int (&arr2)[b]) {} + // since-cxx11-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'dr2233::MultilevelSpecialization::B'}} template<> template<> void B::f<1, 1>(int i, int (&arr1a)[1], int (&arr2a)[1]) {} } @@ -134,10 +143,10 @@ struct C { explicit operator D(); } c; B b1(a); const B &b2{a}; // FIXME ill-formed const B &b3(a); -// expected-error@-1 {{no viable conversion from 'struct A' to 'const B'}} -// expected-note@#dr2267-struct-B {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'struct A' to 'const B &' for 1st argument}} -// expected-note@#dr2267-struct-B {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'struct A' to 'B &&' for 1st argument}} -// expected-note@#dr2267-struct-B {{explicit constructor is not a candidate}} +// since-cxx11-error@-1 {{no viable conversion from 'struct A' to 'const B'}} +// since-cxx11-note@#dr2267-struct-B {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'struct A' to 'const B &' for 1st argument}} +// since-cxx11-note@#dr2267-struct-B {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'struct A' to 'B &&' for 1st argument}} +// since-cxx11-note@#dr2267-struct-B {{explicit constructor is not a candidate}} D d1(c); const D &d2{c}; // FIXME ill-formed diff --git a/clang/test/CXX/drs/dr2354.cpp b/clang/test/CXX/drs/dr2354.cpp deleted file mode 100644 index 3efb0ba555669071aa5ae1ff473a2f01825aa5e3..0000000000000000000000000000000000000000 --- a/clang/test/CXX/drs/dr2354.cpp +++ /dev/null @@ -1,10 +0,0 @@ -// RUN: %clang_cc1 -x c++ -verify %s - -// dr2354: 15 - -namespace DR2354 { - -enum alignas(64) A {}; // expected-error {{'alignas' attribute cannot be applied to an enumeration}} -enum struct alignas(64) B {}; // expected-error {{'alignas' attribute cannot be applied to an enumeration}} - -} // namespace DR2354 diff --git a/clang/test/CXX/drs/dr2390.cpp b/clang/test/CXX/drs/dr2390.cpp index d8ab1e9a1b3853bd60f38a186fe8294c334a776e..3931365b568cebfc2a87a2c9278fda69b84796f8 100644 --- a/clang/test/CXX/drs/dr2390.cpp +++ b/clang/test/CXX/drs/dr2390.cpp @@ -1,6 +1,6 @@ // RUN: %clang_cc1 -E -P %s -o - | FileCheck %s -// dr2390: yes +// dr2390: 14 namespace PR48462 { // Test that macro expansion of the builtin argument works. diff --git a/clang/test/CXX/drs/dr23xx.cpp b/clang/test/CXX/drs/dr23xx.cpp index 6cb10067739f8eb415439cd964b57ef2b4e781cf..9ced61d2aae30dc5e4ebf0239802f9d4dbf419ce 100644 --- a/clang/test/CXX/drs/dr23xx.cpp +++ b/clang/test/CXX/drs/dr23xx.cpp @@ -1,9 +1,10 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++98 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,since-cxx11,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx11,since-cxx14,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx11,since-cxx14,since-cxx17,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11,since-cxx14,since-cxx17,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx11,since-cxx14,since-cxx17,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s #if __cplusplus >= 201103L namespace dr2303 { // dr2303: 12 @@ -14,8 +15,14 @@ struct A<> {}; template struct A : A {}; struct B : A {}; -struct C : A, A {}; // expected-warning {{direct base 'A' is inaccessible}} -struct D : A, A {}; // expected-warning {{direct base 'A' is inaccessible}} +struct C : A, A {}; +/* since-cxx11-warning@-1 {{direct base 'A' is inaccessible due to ambiguity: + struct dr2303::C -> A -> A + struct dr2303::C -> A}} */ +struct D : A, A {}; +/* since-cxx11-warning@-1 {{direct base 'A' is inaccessible due to ambiguity: + struct dr2303::D -> A + struct dr2303::D -> A -> A}} */ struct E : A {}; struct F : B, E {}; @@ -32,7 +39,10 @@ void g() { f2(&b); f(C{}); f(D{}); - f(F{}); // expected-error {{ambiguous conversion from derived class}} + f(F{}); + /* since-cxx11-error@-1 {{ambiguous conversion from derived class 'const F' to base class 'const A': + struct dr2303::F -> B -> A + struct dr2303::F -> E -> A}} */ } } //namespace dr2303 #endif @@ -65,8 +75,10 @@ namespace dr2352 { // dr2352: 10 int *const *const &f2() { return p; } int **const &f3() { return p; } - const int **const &f4() { return p; } // expected-error {{reference to type 'const int **const' could not bind to an lvalue of type 'int **'}} - const int *const *&f5() { return p; } // expected-error {{binding reference of type 'const int *const *' to value of type 'int **' not permitted due to incompatible qualifiers}} + const int **const &f4() { return p; } + // expected-error@-1 {{reference to type 'const int **const' could not bind to an lvalue of type 'int **'}} + const int *const *&f5() { return p; } + // expected-error@-1 {{binding reference of type 'const int *const *' to value of type 'int **' not permitted due to incompatible qualifiers}} // FIXME: We permit this as a speculative defect resolution, allowing // qualification conversions when forming a glvalue conditional expression. @@ -76,7 +88,8 @@ namespace dr2352 { // dr2352: 10 // FIXME: Should we compute the composite pointer type here and produce an // lvalue of type 'const int *const * const'? const int * const * r; - void *y = &(true ? p : r); // expected-error {{rvalue of type 'const int *const *'}} + void *y = &(true ? p : r); + // expected-error@-1 {{rvalue of type 'const int *const *'}} // FIXME: We order these as a speculative defect resolution. void f(const int * const * const &r); @@ -124,12 +137,22 @@ namespace dr2353 { // dr2353: 9 #pragma clang __debug dump not_use_2 } +namespace dr2354 { // dr2354: 15 +#if __cplusplus >= 201103L +enum alignas(64) A {}; +// since-cxx11-error@-1 {{'alignas' attribute cannot be applied to an enumeration}} +enum struct alignas(64) B {}; +// since-cxx11-error@-1 {{'alignas' attribute cannot be applied to an enumeration}} +#endif +} // namespace dr2354 + #if __cplusplus >= 201402L namespace dr2358 { // dr2358: 16 void f2() { int i = 1; void g1(int = [xxx=1] { return xxx; }()); // OK - void g2(int = [xxx=i] { return xxx; }()); // expected-error {{default argument references local variable 'i' of enclosing function}} + void g2(int = [xxx=i] { return xxx; }()); + // since-cxx14-error@-1 {{default argument references local variable 'i' of enclosing function}} } } #endif @@ -148,7 +171,7 @@ class C { }; } // namespace dr2370 -#if __cplusplus >= 201707L +#if __cplusplus >= 201702L // Otherwise, if the qualified-id std::tuple_size names a complete class // type **with a member value**, the expression std::tuple_size::value shall // be a well-formed integral constant expression @@ -165,7 +188,8 @@ template <> struct std::tuple_size { } // namespace std namespace dr2386 { void no_value() { auto [x, y] = Bad1(); } -void wrong_value() { auto [x, y] = Bad2(); } // expected-error {{decomposes into 42 elements}} +void wrong_value() { auto [x, y] = Bad2(); } +// since-cxx17-error@-1 {{type 'Bad2' decomposes into 42 elements, but only 2 names were provided}} } // namespace dr2386 #endif @@ -177,7 +201,8 @@ namespace dr2387 { // dr2387: 9 extern template int a<0>; // ok template static int b = 0; - extern template int b<0>; // expected-error {{internal linkage}} + extern template int b<0>; + // since-cxx14-error@-1 {{explicit instantiation declaration of 'b<0>' with internal linkage}} template const int c = 0; extern template const int c<0>; // ok, has external linkage despite 'const' diff --git a/clang/test/CXX/drs/dr2406.cpp b/clang/test/CXX/drs/dr2406.cpp deleted file mode 100644 index 7ea0870fb70b380216aa5df74b9d4da813502d3e..0000000000000000000000000000000000000000 --- a/clang/test/CXX/drs/dr2406.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// RUN: %clang_cc1 -x c++ %s -verify - -// dr2406: yes - -void fallthrough(int n) { - void g(), h(), i(); - switch (n) { - case 1: - case 2: - g(); - [[fallthrough]]; - case 3: // warning on fallthrough discouraged - do { - [[fallthrough]]; // expected-error {{fallthrough annotation does not directly precede switch label}} - } while (false); - case 6: - do { - [[fallthrough]]; // expected-error {{fallthrough annotation does not directly precede switch label}} - } while (n); - case 7: - while (false) { - [[fallthrough]]; // expected-error {{fallthrough annotation does not directly precede switch label}} - } - case 5: - h(); - case 4: // implementation may warn on fallthrough - i(); - [[fallthrough]]; // expected-error {{fallthrough annotation does not directly precede switch label}} - } -} diff --git a/clang/test/CXX/drs/dr24xx.cpp b/clang/test/CXX/drs/dr24xx.cpp index 3fd8539be53d810d251bcd4b13a2fa99dc9f882d..b34ceb420788fcd7a71669fa558f87c24eb12444 100644 --- a/clang/test/CXX/drs/dr24xx.cpp +++ b/clang/test/CXX/drs/dr24xx.cpp @@ -1,9 +1,52 @@ -// RUN: %clang_cc1 -std=c++20 %s -verify -// RUN: %clang_cc1 -std=c++23 %s -verify +// RUN: %clang_cc1 -std=c++98 %s -verify=expected +// RUN: %clang_cc1 -std=c++11 %s -verify=expected +// RUN: %clang_cc1 -std=c++14 %s -verify=expected +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx17 +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx17 +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx17 +// RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx17 + +#if __cplusplus <= 201402L // expected-no-diagnostics +#endif + +namespace dr2406 { // dr2406: 5 +#if __cplusplus >= 201703L +void fallthrough(int n) { + void g(), h(), i(); + switch (n) { + case 1: + case 2: + g(); + [[fallthrough]]; + case 3: // warning on fallthrough discouraged + do { + [[fallthrough]]; + // since-cxx17-error@-1 {{fallthrough annotation does not directly precede switch label}} + } while (false); + case 6: + do { + [[fallthrough]]; + // since-cxx17-error@-1 {{fallthrough annotation does not directly precede switch label}} + } while (n); + case 7: + while (false) { + [[fallthrough]]; + // since-cxx17-error@-1 {{fallthrough annotation does not directly precede switch label}} + } + case 5: + h(); + case 4: // implementation may warn on fallthrough + i(); + [[fallthrough]]; + // since-cxx17-error@-1 {{fallthrough annotation does not directly precede switch label}} + } +} +#endif +} namespace dr2450 { // dr2450: 18 drafting -#if __cplusplus > 202002L +#if __cplusplus >= 202302L struct S {int a;}; template void f(){} @@ -17,7 +60,7 @@ f<{.a= 0}>(); } namespace dr2459 { // dr2459: 18 drafting -#if __cplusplus > 202002L +#if __cplusplus >= 202302L struct A { constexpr A(float) {} }; diff --git a/clang/test/CXX/drs/dr25xx.cpp b/clang/test/CXX/drs/dr25xx.cpp index 3644e4c328b1bcbcab69b5d494eb6794d50e05b2..8c34b03c22d5b1af7c1340fc1d819f5ca1c748b7 100644 --- a/clang/test/CXX/drs/dr25xx.cpp +++ b/clang/test/CXX/drs/dr25xx.cpp @@ -1,23 +1,23 @@ -// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors - -#if __cplusplus < 201103L +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20,since-cxx23 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20,since-cxx23 -fexceptions -fcxx-exceptions -pedantic-errors + +#if __cplusplus == 199711L // expected-no-diagnostics #endif -namespace dr2516 { // dr2516: yes +namespace dr2516 { // dr2516: 3.0 // NB: reusing 1482 test #if __cplusplus >= 201103L template struct S { typedef char I; }; enum E2 : S::I { e }; -// expected-error@-1 {{use of undeclared identifier 'E2'}} +// since-cxx11-error@-1 {{use of undeclared identifier 'E2'}} #endif } // namespace dr2516 @@ -27,24 +27,24 @@ namespace dr2518 { // dr2518: 17 template void f(T t) { if constexpr (sizeof(T) != sizeof(int)) { -#if __cplusplus < 201703L -// expected-error@-2 {{constexpr if is a C++17 extension}} -#endif - static_assert(false, "must be int-sized"); // expected-error {{must be int-size}} + // cxx11-14-error@-1 {{constexpr if is a C++17 extension}} + static_assert(false, "must be int-sized"); + // since-cxx11-error@-1 {{static assertion failed: must be int-sized}} + // since-cxx11-note@#dr2518-f-c {{in instantiation of function template specialization 'dr2518::f' requested here}} } } void g(char c) { f(0); - f(c); // expected-note {{requested here}} + f(c); // #dr2518-f-c } template struct S { - static_assert(false); // expected-error {{static assertion failed}} -#if __cplusplus < 201703L -// expected-error@-2 {{'static_assert' with no message is a C++17 extension}} -#endif + static_assert(false); + // cxx11-14-error@-1 {{'static_assert' with no message is a C++17 extension}} + // since-cxx11-error@-2 {{static assertion failed}} + // since-cxx11-note@#dr2518-S-double {{in instantiation of template class 'dr2518::S' requested here}} }; template <> @@ -56,7 +56,7 @@ struct S {}; int test_specialization() { S s1; S s2; - S s3; // expected-note {{in instantiation of template class 'dr2518::S' requested here}} + S s3; // #dr2518-S-double } #endif @@ -67,16 +67,16 @@ namespace dr2521 { // dr2521: 17 #pragma clang diagnostic push #pragma clang diagnostic warning "-Wdeprecated-literal-operator" long double operator"" _\u03C0___(long double); -// expected-warning@-1 {{identifier '_π___' preceded by whitespace in a literal operator declaration is deprecated}} -// expected-warning@-2 {{user-defined literal suffixes containing '__' are reserved}} +// since-cxx11-warning@-1 {{identifier '_π___' preceded by whitespace in a literal operator declaration is deprecated}} +// since-cxx11-warning@-2 {{user-defined literal suffixes containing '__' are reserved}} template decltype(sizeof 0) operator"" _div(); -// expected-warning@-1 {{identifier '_div' preceded by whitespace in a literal operator declaration is deprecated}} +// since-cxx11-warning@-1 {{identifier '_div' preceded by whitespace in a literal operator declaration is deprecated}} using ::dr2521::operator"" _\u03C0___; using ::dr2521::operator""_div; -// expected-warning@-2 {{identifier '_π___' preceded by whitespace in a literal operator declaration is deprecated}} +// since-cxx11-warning@-2 {{identifier '_π___' preceded by whitespace in a literal operator declaration is deprecated}} #pragma clang diagnostic pop #endif } // namespace dr2521 @@ -85,12 +85,16 @@ using ::dr2521::operator""_div; #if __cplusplus >= 202302L namespace dr2553 { // dr2553: 18 struct B { - virtual void f(this B&); // expected-error {{an explicit object parameter cannot appear in a virtual function}} - static void f(this B&); // expected-error {{an explicit object parameter cannot appear in a static function}} - virtual void g(); // expected-note {{here}} + virtual void f(this B&); + // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}} + static void f(this B&); + // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a static function}} + virtual void g(); // #dr2553-g }; struct D : B { - void g(this D&); // expected-error {{an explicit object parameter cannot appear in a virtual function}} + void g(this D&); + // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}} + // since-cxx23-note@#dr2553-g {{overridden virtual function is here}} }; } @@ -99,19 +103,25 @@ struct D : B { #if __cplusplus >= 202302L namespace dr2554 { // dr2554: 18 review struct B { - virtual void f(); // expected-note 3{{here}} + virtual void f(); // #dr2554-g }; struct D : B { - void f(this D&); // expected-error {{an explicit object parameter cannot appear in a virtual function}} + void f(this D&); + // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}} + // since-cxx23-note@#dr2554-g {{overridden virtual function is here}} }; struct D2 : B { - void f(this B&); // expected-error {{an explicit object parameter cannot appear in a virtual function}} + void f(this B&); + // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}} + // since-cxx23-note@#dr2554-g {{overridden virtual function is here}} }; struct T {}; struct D3 : B { - void f(this T&); // expected-error {{an explicit object parameter cannot appear in a virtual function}} + void f(this T&); + // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}} + // since-cxx23-note@#dr2554-g {{overridden virtual function is here}} }; } @@ -153,48 +163,48 @@ namespace dr2565 { // dr2565: 16 static_assert(is_referenceable::value); template - concept TwoParams = requires (T *a, U b){ true;}; // #TPC + concept TwoParams = requires (T *a, U b){ true;}; // #dr2565-TPC template - requires TwoParams // #TPSREQ + requires TwoParams // #dr2565-TPSREQ struct TwoParamsStruct{}; using TPSU = TwoParamsStruct; - // expected-error@-1{{constraints not satisfied for class template 'TwoParamsStruct'}} - // expected-note@#TPSREQ{{because 'TwoParams' evaluated to false}} - // expected-note@#TPC{{because 'b' would be invalid: argument may not have 'void' type}} + // since-cxx20-error@-1 {{constraints not satisfied for class template 'TwoParamsStruct'}} + // since-cxx20-note@#dr2565-TPSREQ {{because 'TwoParams' evaluated to false}} + // since-cxx20-note@#dr2565-TPC {{because 'b' would be invalid: argument may not have 'void' type}} template - concept Variadic = requires (U* ... a, T b){ true;}; // #VC + concept Variadic = requires (U* ... a, T b){ true;}; // #dr2565-VC template - requires Variadic // #VSREQ + requires Variadic // #dr2565-VSREQ struct VariadicStruct{}; using VSU = VariadicStruct; - // expected-error@-1{{constraints not satisfied for class template 'VariadicStruct'}} - // expected-note@#VSREQ{{because 'Variadic' evaluated to false}} - // expected-note@#VC{{because 'b' would be invalid: argument may not have 'void' type}} + // since-cxx20-error@-1 {{constraints not satisfied for class template 'VariadicStruct'}} + // since-cxx20-note@#dr2565-VSREQ {{because 'Variadic' evaluated to false}} + // since-cxx20-note@#dr2565-VC {{because 'b' would be invalid: argument may not have 'void' type}} template - // expected-error@+1 {{unknown type name 'ErrorRequires'}} concept ErrorRequires = requires (ErrorRequires auto x) { + // since-cxx20-error@-1 {{unknown type name 'ErrorRequires'}} x; }; static_assert(ErrorRequires); - // expected-error@-1{{static assertion failed}} - // expected-note@-2{{because substituted constraint expression is ill-formed: constraint depends on a previously diagnosed expression}} + // since-cxx20-error@-1 {{static assertion failed}} + // since-cxx20-note@-2 {{because substituted constraint expression is ill-formed: constraint depends on a previously diagnosed expression}} template - // expected-error@+2 {{unknown type name 'NestedErrorInRequires'}} concept NestedErrorInRequires = requires (T x) { requires requires (NestedErrorInRequires auto y) { + // since-cxx20-error@-1 {{unknown type name 'NestedErrorInRequires'}} y; }; }; static_assert(NestedErrorInRequires); - // expected-error@-1{{static assertion failed}} - // expected-note@-2{{because substituted constraint expression is ill-formed: constraint depends on a previously diagnosed expression}} + // expected-error@-1 {{static assertion failed}} + // expected-note@-2 {{because substituted constraint expression is ill-formed: constraint depends on a previously diagnosed expression}} #endif } diff --git a/clang/test/CXX/drs/dr26xx.cpp b/clang/test/CXX/drs/dr26xx.cpp index 8517cd5872b183dec666244c681f9e944621c670..dd4bb1ff6ae2e1ed2b2ad15262f9f73fba2826bf 100644 --- a/clang/test/CXX/drs/dr26xx.cpp +++ b/clang/test/CXX/drs/dr26xx.cpp @@ -1,8 +1,14 @@ -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -// RUN: %clang_cc1 -std=c++2b -triple x86_64-unknown-unknown %s -verify +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,cxx11 +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20 +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20,since-cxx23 +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20,since-cxx23 -namespace dr2621 { // dr2621: yes +namespace dr2621 { // dr2621: 16 +#if __cplusplus >= 202002L enum class E { a }; namespace One { using E_t = E; @@ -12,33 +18,39 @@ auto v = a; namespace Two { using dr2621::E; int E; // we see this -using enum E; // expected-error {{unknown type name E}} +using enum E; +// since-cxx20-error@-1 {{unknown type name E}} } +#endif } namespace dr2628 { // dr2628: no open // this was reverted for the 16.x release // due to regressions, see the issue for more details: // https://github.com/llvm/llvm-project/issues/60777 - +#if __cplusplus >= 202002L template struct foo { // The expected notes below should be removed when dr2628 is fully implemented again - constexpr foo() requires (!A && !B) = delete; // expected-note {{candidate function [with A = false, B = false]}} #DR2628_CTOR - constexpr foo() requires (A || B) = delete; // expected-note {{candidate function [with A = false, B = false]}} + constexpr foo() requires (!A && !B) = delete; // #dr2628-ctor-1 + constexpr foo() requires (A || B) = delete; // #dr2628-ctor-2 }; void f() { // The FIXME's below should be the expected errors when dr2628 is // fully implemented again. - // FIXME-expected-error {{call to deleted}} - foo fooable; // expected-error {{ambiguous deduction for template arguments of 'foo'}} - // FIXME-expected-note@#DR2628_CTOR {{marked deleted here}} + foo fooable; // #dr2628-fooable + // since-cxx20-error@-1 {{ambiguous deduction for template arguments of 'foo'}} + // since-cxx20-note@#dr2628-ctor-1 {{candidate function [with A = false, B = false]}} + // since-cxx20-note@#dr2628-ctor-2 {{candidate function [with A = false, B = false]}} + // FIXME-since-cxx20-error@#dr2628-fooable {{call to deleted}} + // FIXME-since-cxx20-note@#dr2628-ctor {{marked deleted here}} } - +#endif } namespace dr2631 { // dr2631: 16 +#if __cplusplus >= 202002L constexpr int g(); consteval int f() { return g(); @@ -52,9 +64,11 @@ namespace dr2631 { // dr2631: 16 int test() { return k(); } +#endif } namespace dr2635 { // dr2635: 16 +#if __cplusplus >= 202002L template concept UnaryC = true; template @@ -67,66 +81,79 @@ template T get_T(); void use() { - // expected-error@+1{{decomposition declaration cannot be declared with constrained 'auto'}} UnaryC auto [a, b] = get_S(); - // expected-error@+1{{decomposition declaration cannot be declared with constrained 'auto'}} + // since-cxx20-error@-1 {{decomposition declaration cannot be declared with constrained 'auto'}} BinaryC auto [c, d] = get_S(); + // since-cxx20-error@-1 {{decomposition declaration cannot be declared with constrained 'auto'}} } template void TemplUse() { - // expected-error@+1{{decomposition declaration cannot be declared with constrained 'auto'}} UnaryC auto [a, b] = get_T(); - // expected-error@+1{{decomposition declaration cannot be declared with constrained 'auto'}} + // since-cxx20-error@-1 {{decomposition declaration cannot be declared with constrained 'auto'}} BinaryC auto [c, d] = get_T(); + // since-cxx20-error@-1 {{decomposition declaration cannot be declared with constrained 'auto'}} } +#endif } - // dr2636: na +// dr2636: na namespace dr2640 { // dr2640: 16 -int \N{Λ} = 0; //expected-error {{'Λ' is not a valid Unicode character name}} \ - //expected-error {{expected unqualified-id}} -const char* emoji = "\N{🤡}"; // expected-error {{'🤡' is not a valid Unicode character name}} \ - // expected-note 5{{did you mean}} +int \N{Λ} = 0; +// expected-error@-1 {{'Λ' is not a valid Unicode character name}} +// expected-error@-2 {{expected unqualified-id}} +const char* emoji = "\N{🤡}"; +// expected-error@-1 {{'🤡' is not a valid Unicode character name}} +// expected-note@-2 {{did you mean OX ('🐂' U+1F402)?}} +// expected-note@-3 {{did you mean ANT ('🐜' U+1F41C)?}} +// expected-note@-4 {{did you mean ARC ('⌒' U+2312)?}} +// expected-note@-5 {{did you mean AXE ('🪓' U+1FA93)?}} +// expected-note@-6 {{did you mean BAT ('🦇' U+1F987)?}} #define z(x) 0 #define dr2640_a z( -int x = dr2640_a\N{abc}); // expected-error {{'abc' is not a valid Unicode character name}} -int y = dr2640_a\N{LOTUS}); // expected-error {{character not allowed in an identifier}} \ - // expected-error {{use of undeclared identifier 'dr2640_a🪷'}} \ - // expected-error {{extraneous ')' before ';'}} -} - - // dr2642: na - -namespace dr2644 { // dr2644: yes - -auto z = [a = 42](int a) { // expected-error {{a lambda parameter cannot shadow an explicitly captured entity}} \ - // expected-note {{variable 'a' is explicitly captured here}} +int x = dr2640_a\N{abc}); +// expected-error@-1 {{'abc' is not a valid Unicode character name}} +int y = dr2640_a\N{LOTUS}); +// expected-error@-1 {{character not allowed in an identifier}} +// expected-error@-2 {{use of undeclared identifier 'dr2640_a🪷'}} +// expected-error@-3 {{extraneous ')' before ';'}} +} + +// dr2642: na + +namespace dr2644 { // dr2644: 8 +#if __cplusplus >= 201103L +auto z = [a = 42](int a) { +// cxx11-warning@-1 {{initialized lambda captures are a C++14 extension}} +// since-cxx11-error@-2 {{a lambda parameter cannot shadow an explicitly captured entity}} +// since-cxx11-note@-3 {{variable 'a' is explicitly captured here}} return 1; }; - +#endif } #if __cplusplus >= 202302L -namespace dr2650 { // dr2650: yes +namespace dr2650 { // dr2650: 17 template struct S {}; -template int f(S*); // expected-note {{type 'X' of non-type template parameter is not a structural type}} +template int f(S*); // #dr2650-f class X { int m; }; -int i0 = f(0); //expected-error {{no matching function for call to 'f'}} +int i0 = f(0); +// since-cxx23-error@-1 {{no matching function for call to 'f'}} +// since-cxx23-note@#dr2650-f {{type 'X' of non-type template parameter is not a structural type}} } #endif #if __cplusplus >= 202302L namespace dr2653 { // dr2653: 18 struct Test { void f(this const auto& = Test{}); }; - // expected-error@-1 {{the explicit object parameter cannot have a default argument}} + // since-cxx23-error@-1 {{the explicit object parameter cannot have a default argument}} auto L = [](this const auto& = Test{}){}; - // expected-error@-1 {{the explicit object parameter cannot have a default argument}} + // since-cxx23-error@-1 {{the explicit object parameter cannot have a default argument}} } #endif @@ -141,6 +168,7 @@ void f() { } namespace dr2681 { // dr2681: 17 +#if __cplusplus >= 202002L using size_t = decltype(sizeof(int)); template @@ -152,7 +180,7 @@ struct I { volatile T array[N]; }; template -struct J { // expected-note 3{{candidate}} +struct J { // #dr2681-J unsigned char array[N]; }; @@ -161,15 +189,24 @@ I i = { "def" }; static_assert(__is_same(decltype(h), H)); // Not H static_assert(__is_same(decltype(i), I)); -J j = { "ghi" }; // expected-error {{no viable constructor or deduction guide}} +J j = { "ghi" }; +// since-cxx20-error@-1 {{no viable constructor or deduction guide}} +// since-cxx20-note@#dr2681-J {{candidate template ignored: could not match 'J' against 'const char *'}} +// since-cxx20-note@#dr2681-J {{candidate template ignored: could not match 'const unsigned char' against 'const char'}} +// since-cxx20-note@#dr2681-J {{candidate function template not viable: requires 0 arguments, but 1 was provided}} +#endif } namespace dr2672 { // dr2672: 18 open +#if __cplusplus >= 202002L template -void f(T) requires requires { []() { T::invalid; } (); }; // expected-error{{type 'int' cannot be used prior to '::'}} - // expected-note@-1{{while substituting into a lambda expression here}} - // expected-note@-2{{in instantiation of requirement here}} - // expected-note@-3{{while substituting template arguments into constraint expression here}} +void f(T) requires requires { []() { T::invalid; } (); }; +// since-cxx20-error@-1 {{type 'int' cannot be used prior to '::' because it has no members}} +// since-cxx20-note@-2 {{while substituting into a lambda expression here}} +// since-cxx20-note@-3 {{in instantiation of requirement here}} +// since-cxx20-note@-4 {{while substituting template arguments into constraint expression here}} +// since-cxx20-note@#dr2672-f-0 {{while checking constraint satisfaction for template 'f' required here}} +// since-cxx20-note@#dr2672-f-0 {{in instantiation of function template specialization 'dr2672::f' requested here}} void f(...); template @@ -179,11 +216,12 @@ void bar(T) requires requires { void bar(...); void m() { - f(0); // expected-note {{while checking constraint satisfaction for template 'f' required here}} - // expected-note@-1 {{in instantiation of function template specialization}} + f(0); // #dr2672-f-0 bar(0); } +#endif } + #if __cplusplus >= 202302L namespace dr2687 { // dr2687: 18 struct S{ @@ -193,7 +231,8 @@ struct S{ }; void test() { - (&S::f)(1); // expected-error {{called object type 'void (dr2687::S::*)(int)' is not a function or function pointer}} + (&S::f)(1); + // since-cxx23-error@-1 {{called object type 'void (dr2687::S::*)(int)' is not a function or function pointer}} (&S::g)(1); (&S::h)(S(), 1); } diff --git a/clang/test/CXX/drs/dr27xx.cpp b/clang/test/CXX/drs/dr27xx.cpp index 5c7ce98f878da6b3679ff0e488634abd70005f86..4f7d0d6b44a83ec7844f1ccf5221158d9f23e9e8 100644 --- a/clang/test/CXX/drs/dr27xx.cpp +++ b/clang/test/CXX/drs/dr27xx.cpp @@ -1,6 +1,17 @@ -// RUN: %clang_cc1 -std=c++2c -verify %s +// RUN: %clang_cc1 -std=c++98 -verify=expected %s +// RUN: %clang_cc1 -std=c++11 -verify=expected %s +// RUN: %clang_cc1 -std=c++14 -verify=expected %s +// RUN: %clang_cc1 -std=c++17 -verify=expected %s +// RUN: %clang_cc1 -std=c++20 -verify=expected %s +// RUN: %clang_cc1 -std=c++23 -verify=expected,since-cxx23 %s +// RUN: %clang_cc1 -std=c++2c -verify=expected,since-cxx23,since-cxx26 %s + +#if __cplusplus <= 202002L +// expected-no-diagnostics +#endif namespace dr2789 { // dr2789: 18 open +#if __cplusplus >= 202302L template struct Base { constexpr void g(); // #dr2789-g1 @@ -23,11 +34,12 @@ struct S : Base, Base2 { void test() { S<> s; s.f(); - s.g(); // expected-error {{call to member function 'g' is ambiguous}} - // expected-note@#dr2789-g1 {{candidate function}} - // expected-note@#dr2789-g2 {{candidate function}} + s.g(); + // since-cxx23-error@-1 {{call to member function 'g' is ambiguous}} + // since-cxx23-note@#dr2789-g1 {{candidate function}} + // since-cxx23-note@#dr2789-g2 {{candidate function}} } - +#endif } namespace dr2798 { // dr2798: 17 drafting @@ -49,7 +61,8 @@ struct X { }; consteval X f() { return {}; } -static_assert(false, f().s); // expected-error {{static assertion failed: Hello}} +static_assert(false, f().s); +// since-cxx26-error@-1 {{static assertion failed: Hello}} #endif } // namespace dr2798 diff --git a/clang/test/CXX/drs/dr412.cpp b/clang/test/CXX/drs/dr412.cpp index 27bc7e5cf94e0cfe86a79b666a6fec5d223e7432..8ea29135d1df825481817858bf7f87cb3df6f8d7 100644 --- a/clang/test/CXX/drs/dr412.cpp +++ b/clang/test/CXX/drs/dr412.cpp @@ -1,9 +1,12 @@ // RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT="throw()" -DBAD_ALLOC="throw(std::bad_alloc)" // RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC= // RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC= -// RUN: %clang_cc1 -std=c++1z %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC= +// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC= +// RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC= +// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC= +// RUN: %clang_cc1 -std=c++2c %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC= -// dr412: yes +// dr412: 3.4 // lwg404: yes // lwg2340: yes @@ -11,11 +14,17 @@ __extension__ typedef __SIZE_TYPE__ size_t; namespace std { struct bad_alloc {}; } -inline void* operator new(size_t) BAD_ALLOC; // expected-error {{cannot be declared 'inline'}} -inline void* operator new[](size_t) BAD_ALLOC; // expected-error {{cannot be declared 'inline'}} -inline void operator delete(void*) NOEXCEPT; // expected-error {{cannot be declared 'inline'}} -inline void operator delete[](void*) NOEXCEPT; // expected-error {{cannot be declared 'inline'}} +inline void* operator new(size_t) BAD_ALLOC; +// expected-error@-1 {{replacement function 'operator new' cannot be declared 'inline'}} +inline void* operator new[](size_t) BAD_ALLOC; +// expected-error@-1 {{replacement function 'operator new[]' cannot be declared 'inline'}} +inline void operator delete(void*) NOEXCEPT; +// expected-error@-1 {{replacement function 'operator delete' cannot be declared 'inline'}} +inline void operator delete[](void*) NOEXCEPT; +// expected-error@-1 {{replacement function 'operator delete[]' cannot be declared 'inline'}} #ifdef __cpp_sized_deallocation -inline void operator delete(void*, size_t) NOEXCEPT; // expected-error {{cannot be declared 'inline'}} -inline void operator delete[](void*, size_t) NOEXCEPT; // expected-error {{cannot be declared 'inline'}} +inline void operator delete(void*, size_t) NOEXCEPT; +// expected-error@-1 {{replacement function 'operator delete' cannot be declared 'inline'}} +inline void operator delete[](void*, size_t) NOEXCEPT; +// expected-error@-1 {{replacement function 'operator delete[]' cannot be declared 'inline'}} #endif diff --git a/clang/test/CXX/drs/dr7xx.cpp b/clang/test/CXX/drs/dr7xx.cpp index 11901b80d64622555649076cfb665de4842f48e8..926bff1cc479c5c7d0f9ce7597dd318b7b145422 100644 --- a/clang/test/CXX/drs/dr7xx.cpp +++ b/clang/test/CXX/drs/dr7xx.cpp @@ -1,37 +1,53 @@ -// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++2a %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++98 %s -verify=expected,cxx98-14,cxx98-11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 %s -verify=expected,cxx98-14,cxx98-11,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++14 %s -verify=expected,cxx98-14,since-cxx14,since-cxx11,cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++17 %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++2a %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors namespace dr705 { // dr705: yes namespace N { struct S {}; - void f(S); // expected-note {{declared here}} + void f(S); // #dr705-f } void g() { N::S s; f(s); // ok - (f)(s); // expected-error {{use of undeclared}} + (f)(s); + // expected-error@-1 {{use of undeclared identifier 'f'}} + // expected-note@#dr705-f {{'N::f' declared here}} } } namespace dr712 { // dr712: partial void use(int); void f() { - const int a = 0; // expected-note 5{{here}} + const int a = 0; // #dr712-f-a struct X { void g(bool cond) { use(a); use((a)); use(cond ? a : a); - use((cond, a)); // expected-warning 2{{left operand of comma operator has no effect}} FIXME: should only warn once - - (void)a; // FIXME: expected-error {{declared in enclosing}} - (void)(a); // FIXME: expected-error {{declared in enclosing}} - (void)(cond ? a : a); // FIXME: expected-error 2{{declared in enclosing}} - (void)(cond, a); // FIXME: expected-error {{declared in enclosing}} expected-warning {{left operand of comma operator has no effect}} + // FIXME: should only warn once + use((cond, a)); + // expected-warning@-1 {{left operand of comma operator has no effect}} + // expected-warning@-2 {{left operand of comma operator has no effect}} + + (void)a; + // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME + // expected-note@#dr712-f-a {{'a' declared here}} + (void)(a); + // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME + // expected-note@#dr712-f-a {{'a' declared here}} + (void)(cond ? a : a); // #dr712-ternary + // expected-error@#dr712-ternary {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME + // expected-note@#dr712-f-a {{'a' declared here}} + // expected-error@#dr712-ternary {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME + // expected-note@#dr712-f-a {{'a' declared here}} + (void)(cond, a); // #dr712-comma + // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME + // expected-note@#dr712-f-a {{'a' declared here}} + // expected-warning@#dr712-comma {{left operand of comma operator has no effect}} } }; } @@ -39,14 +55,18 @@ namespace dr712 { // dr712: partial #if __cplusplus >= 201103L void g() { struct A { int n; }; - constexpr A a = {0}; // expected-note 2{{here}} + constexpr A a = {0}; // #dr712-g-a struct X { void g(bool cond) { use(a.n); use(a.*&A::n); - (void)a.n; // FIXME: expected-error {{declared in enclosing}} - (void)(a.*&A::n); // FIXME: expected-error {{declared in enclosing}} + (void)a.n; + // since-cxx11-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::g'}} FIXME + // since-cxx11-note@#dr712-g-a {{'a' declared here}} + (void)(a.*&A::n); + // since-cxx11-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::g'}} FIXME + // since-cxx11-note@#dr712-g-a {{'a' declared here}} } }; } @@ -55,9 +75,10 @@ namespace dr712 { // dr712: partial namespace dr727 { // dr727: partial struct A { - template struct C; // expected-note 6{{here}} - template void f(); // expected-note {{here}} - template static int N; // expected-error 0-1{{C++14}} expected-note 6{{here}} + template struct C; // #dr727-C + template void f(); // #dr727-f + template static int N; // #dr727-N + // cxx98-11-error@-1 {{variable templates are a C++14 extension}} template<> struct C; template<> void f(); @@ -67,19 +88,40 @@ namespace dr727 { // dr727: partial template static int N; struct B { - template<> struct C; // expected-error {{not in class 'A' or an enclosing namespace}} - template<> void f(); // expected-error {{no function template matches}} - template<> static int N; // expected-error {{not in class 'A' or an enclosing namespace}} - - template struct C; // expected-error {{not in class 'A' or an enclosing namespace}} - template static int N; // expected-error {{not in class 'A' or an enclosing namespace}} - - template<> struct A::C; // expected-error {{not in class 'A' or an enclosing namespace}} - template<> void A::f(); // expected-error {{no function template matches}} expected-error {{cannot have a qualified name}} - template<> static int A::N; // expected-error {{not in class 'A' or an enclosing namespace}} expected-error {{cannot have a qualified name}} - - template struct A::C; // expected-error {{not in class 'A' or an enclosing namespace}} - template static int A::N; // expected-error {{not in class 'A' or an enclosing namespace}} expected-error {{cannot have a qualified name}} + template<> struct C; + // expected-error@-1 {{class template specialization of 'C' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-C {{explicitly specialized declaration is here}} + template<> void f(); + // expected-error@-1 {{no function template matches function template specialization 'f'}} + template<> static int N; + // expected-error@-1 {{variable template specialization of 'N' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-N {{explicitly specialized declaration is here}} + + template struct C; + // expected-error@-1 {{class template partial specialization of 'C' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-C {{explicitly specialized declaration is here}} + template static int N; + // expected-error@-1 {{variable template partial specialization of 'N' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-N {{explicitly specialized declaration is here}} + + template<> struct A::C; + // expected-error@-1 {{class template specialization of 'C' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-C {{explicitly specialized declaration is here}} + template<> void A::f(); + // expected-error@-1 {{o function template matches function template specialization 'f'}} + // expected-error@-2 {{non-friend class member 'f' cannot have a qualified name}} + template<> static int A::N; + // expected-error@-1 {{non-friend class member 'N' cannot have a qualified name}} + // expected-error@-2 {{variable template specialization of 'N' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-N {{explicitly specialized declaration is here}} + + template struct A::C; + // expected-error@-1 {{class template partial specialization of 'C' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-C {{explicitly specialized declaration is here}} + template static int A::N; + // expected-error@-1 {{non-friend class member 'N' cannot have a qualified name}} + // expected-error@-2 {{variable template partial specialization of 'N' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-N {{explicitly specialized declaration is here}} }; }; @@ -91,19 +133,36 @@ namespace dr727 { // dr727: partial template int A::N; namespace C { - template<> struct A::C; // expected-error {{not in class 'A' or an enclosing namespace}} - template<> void A::f(); // expected-error {{not in class 'A' or an enclosing namespace}} - template<> int A::N; // expected-error {{not in class 'A' or an enclosing namespace}} - - template struct A::C; // expected-error {{not in class 'A' or an enclosing namespace}} - template int A::N; // expected-error {{not in class 'A' or an enclosing namespace}} + template<> struct A::C; + // expected-error@-1 {{class template specialization of 'C' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-C {{explicitly specialized declaration is here}} + template<> void A::f(); + // expected-error@-1 {{function template specialization of 'f' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-f {{explicitly specialized declaration is here}} + template<> int A::N; + // expected-error@-1 {{variable template specialization of 'N' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-N {{explicitly specialized declaration is here}} + + template struct A::C; + // expected-error@-1 {{class template partial specialization of 'C' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-C {{explicitly specialized declaration is here}} + template int A::N; + // expected-error@-1 {{variable template partial specialization of 'N' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-N {{explicitly specialized declaration is here}} } template struct D { - template struct C { typename T::error e; }; // expected-error {{no members}} - template void f() { T::error; } // expected-error {{no members}} - template static const int N = T::error; // expected-error {{no members}} expected-error 0-1{{C++14}} + template struct C { typename T::error e; }; + // expected-error@-1 {{type 'float' cannot be used prior to '::' because it has no members}} + // expected-note@#dr727-C-float {{in instantiation of template class 'dr727::D::C' requested here}} + template void f() { T::error; } + // expected-error@-1 {{type 'float' cannot be used prior to '::' because it has no members}} + // expected-note@#dr727-f-float {{in instantiation of function template specialization 'dr727::D::f' requested here}} + template static const int N = T::error; + // cxx98-11-error@-1 {{variable templates are a C++14 extension}} + // expected-error@-2 {{type 'float' cannot be used prior to '::' because it has no members}} + // expected-note@#dr727-N-float {{in instantiation of static data member 'dr727::D::N' requested here}} template<> struct C {}; template<> void f() {} @@ -114,7 +173,8 @@ namespace dr727 { // dr727: partial template struct E { - template<> void f() {} // expected-error {{no candidate function template}} + template<> void f() {} + // expected-error@-1 {{no candidate function template was found for dependent member function template specialization}} }; }; @@ -126,9 +186,9 @@ namespace dr727 { // dr727: partial D::C(); int b = D::N; - D::C(); // expected-note {{instantiation of}} - di.f(); // expected-note {{instantiation of}} - int c = D::N; // expected-note {{instantiation of}} + D::C(); // #dr727-C-float + di.f(); // #dr727-f-float + int c = D::N; // #dr727-N-float } namespace mixed_inner_outer_specialization { @@ -148,28 +208,30 @@ namespace dr727 { // dr727: partial #if __cplusplus >= 201402L template struct B { template static const int u = 1; - template<> static const int u<0> = 2; // expected-note {{here}} + template<> static const int u<0> = 2; // #dr727-u0 // Note that in C++17 onwards, these are implicitly inline, and so the // initializer of v<0> is not instantiated with the declaration. In // C++14, v<0> is a non-defining declaration and its initializer is // instantiated with the class. template static constexpr int v = 1; - template<> static constexpr int v<0> = 2; // #v0 + template<> static constexpr int v<0> = 2; // #dr727-v0 - template static const inline int w = 1; // expected-error 0-1{{C++17 extension}} - template<> static const inline int w<0> = 2; // expected-error 0-1{{C++17 extension}} + template static const inline int w = 1; + // cxx14-error@-1 {{inline variables are a C++17 extension}} + template<> static const inline int w<0> = 2; + // cxx14-error@-1 {{inline variables are a C++17 extension}} }; template<> template constexpr int B<0>::u = 3; - template<> template<> constexpr int B<0>::u<0> = 4; // expected-error {{already has an initializer}} + template<> template<> constexpr int B<0>::u<0> = 4; + // since-cxx14-error@-1 {{static data member 'u' already has an initializer}} + // since-cxx14-note@#dr727-u0 {{previous initialization is here}} template<> template constexpr int B<0>::v = 3; template<> template<> constexpr int B<0>::v<0> = 4; -#if __cplusplus < 201702L - // expected-error@-2 {{already has an initializer}} - // expected-note@#v0 {{here}} -#endif + // cxx14-error@-1 {{static data member 'v' already has an initializer}} + // cxx14-note@#dr727-v0 {{previous initialization is here}} template<> template constexpr int B<0>::w = 3; template<> template<> constexpr int B<0>::w<0> = 4; @@ -182,10 +244,8 @@ namespace dr727 { // dr727: partial static_assert(B<1>().v<0> == 2, ""); static_assert(B<0>().v<1> == 3, ""); static_assert(B<0>().v<0> == 4, ""); -#if __cplusplus < 201702L - // expected-error@-2 {{failed}} \ - // expected-note@-2 {{evaluates to '2 == 4'}} -#endif + // cxx14-error@-1 {{static assertion failed due to requirement 'dr727::mixed_inner_outer_specialization::B<0>().v<0> == 4'}} + // cxx14-note@-2 {{expression evaluates to '2 == 4'}} static_assert(B<1>().w<1> == 1, ""); static_assert(B<1>().w<0> == 2, ""); @@ -205,13 +265,23 @@ namespace dr727 { // dr727: partial template<> int f2() {} template<> int f2() {} - template static int v1; // expected-error 0-1{{C++14 extension}} - template<> static int v1; // expected-note {{previous}} - template<> static int v1; // expected-error {{duplicate member}} - - template static inline int v2; // expected-error 0-1{{C++17 extension}} expected-error 0-1{{C++14 extension}} - template<> static inline int v2; // expected-error 0-1{{C++17 extension}} expected-note {{previous}} - template<> static inline int v2; // expected-error 0-1{{C++17 extension}} expected-error {{duplicate member}} + template static int v1; + // cxx98-11-error@-1 {{variable templates are a C++14 extension}} + template<> static int v1; // #dr727-v1-T + template<> static int v1; + // expected-error@-1 {{duplicate member 'v1'}} + // expected-note@#dr727-Collision-int-int {{in instantiation of template class 'dr727::Collision' requested here}} + // expected-note@#dr727-v1-T {{previous}} + + template static inline int v2; + // cxx98-11-error@-1 {{variable templates are a C++14 extension}} + // cxx98-14-error@-2 {{inline variables are a C++17 extension}} + template<> static inline int v2; // #dr727-v2-T + // cxx98-14-error@-1 {{inline variables are a C++17 extension}} + template<> static inline int v2; + // cxx98-14-error@-1 {{inline variables are a C++17 extension}} + // expected-error@-2 {{duplicate member 'v2'}} + // expected-note@#dr727-v2-T {{previous declaration is here}} // FIXME: Missing diagnostic for duplicate class explicit specialization. template struct S1; @@ -219,10 +289,12 @@ namespace dr727 { // dr727: partial template<> struct S1; template struct S2; - template<> struct S2 {}; // expected-note {{previous}} - template<> struct S2 {}; // expected-error {{redefinition}} + template<> struct S2 {}; // #dr727-S2-T + template<> struct S2 {}; + // expected-error@-1 {{redefinition of 'S2'}} + // expected-note@#dr727-S2-T {{previous}} }; - Collision c; // expected-note {{in instantiation of}} + Collision c; // #dr727-Collision-int-int } namespace dr777 { // dr777: 3.7 diff --git a/clang/test/CXX/drs/dr8xx.cpp b/clang/test/CXX/drs/dr8xx.cpp index d79430f9e03cae2d94914fea6614a05a178b2c80..b031e65095cdb03c1895e8e23978f5897a620fa4 100644 --- a/clang/test/CXX/drs/dr8xx.cpp +++ b/clang/test/CXX/drs/dr8xx.cpp @@ -1,30 +1,27 @@ -// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +#if __cplusplus == 199711L // expected-no-diagnostics +#endif -namespace dr873 { // dr873: yes +namespace dr873 { // dr873: 3.0 #if __cplusplus >= 201103L template void f(T &&); -template <> void f(int &) {} // #1 -template <> void f(int &&) {} // #2 +template <> void f(int &) = delete; // #dr873-lvalue-ref +template <> void f(int &&) = delete; // #dr873-rvalue-ref void g(int i) { - f(i); // calls f(int&), i.e., #1 -#pragma clang __debug dump f(i) - // CHECK: CallExpr {{.*}} - // CHECK-NEXT: |-ImplicitCastExpr {{.*}} - // CHECK-NEXT: | `-DeclRefExpr {{.*}} 'f' 'void (int &)' {{.*}} - - f(0); // calls f(int&&), i.e., #2 -#pragma clang __debug dump f(0) - // CHECK: CallExpr {{.*}} - // CHECK-NEXT: |-ImplicitCastExpr {{.*}} - // CHECK-NEXT: | `-DeclRefExpr {{.*}} 'f' 'void (int &&)' {{.*}} + f(i); // calls f(int&) + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr873-lvalue-ref {{candidate function [with T = int &] has been implicitly deleted}} + f(0); // calls f(int&&) + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr873-rvalue-ref {{candidate function [with T = int] has been implicitly deleted}} } #endif } // namespace dr873 diff --git a/clang/test/CXX/drs/dr9xx.cpp b/clang/test/CXX/drs/dr9xx.cpp index fb13ef2967f8804c6b49d8ce3995f1e860a2f607..7ea295a684b7a299a192305614da4d27affe0ef5 100644 --- a/clang/test/CXX/drs/dr9xx.cpp +++ b/clang/test/CXX/drs/dr9xx.cpp @@ -1,9 +1,9 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors namespace std { __extension__ typedef __SIZE_TYPE__ size_t; @@ -43,30 +43,34 @@ namespace dr948 { // dr948: 3.7 #endif } -namespace dr952 { // dr952: yes +namespace dr952 { // dr952: 2.8 namespace example1 { struct A { - typedef int I; // #dr952-typedef-decl + typedef int I; // #dr952-I }; -struct B : private A { // #dr952-inheritance +struct B : private A { // #dr952-B }; struct C : B { void f() { - I i1; // expected-error {{private member}} - // expected-note@#dr952-inheritance {{constrained by private inheritance}} - // expected-note@#dr952-typedef-decl {{declared here}} + I i1; + // expected-error@-1 {{'I' is a private member of 'dr952::example1::A'}} + // expected-note@#dr952-B {{constrained by private inheritance here}} + // expected-note@#dr952-I {{member is declared here}} } - I i2; // expected-error {{private member}} - // expected-note@#dr952-inheritance {{constrained by private inheritance}} - // expected-note@#dr952-typedef-decl {{declared here}} + I i2; + // expected-error@-1 {{'I' is a private member of 'dr952::example1::A'}} + // expected-note@#dr952-B {{constrained by private inheritance here}} + // expected-note@#dr952-I {{member is declared here}} struct D { - I i3; // expected-error {{private member}} - // expected-note@#dr952-inheritance {{constrained by private inheritance}} - // expected-note@#dr952-typedef-decl {{declared here}} + I i3; + // expected-error@-1 {{'I' is a private member of 'dr952::example1::A'}} + // expected-note@#dr952-B {{constrained by private inheritance here}} + // expected-note@#dr952-I {{member is declared here}} void g() { - I i4; // expected-error {{private member}} - // expected-note@#dr952-inheritance {{constrained by private inheritance}} - // expected-note@#dr952-typedef-decl {{declared here}} + I i4; + // expected-error@-1 {{'I' is a private member of 'dr952::example1::A'}} + // expected-note@#dr952-B {{constrained by private inheritance here}} + // expected-note@#dr952-I {{member is declared here}} } }; }; @@ -91,10 +95,10 @@ namespace dr974 { // dr974: yes } namespace dr977 { // dr977: yes -enum E { e = E() }; +enum E { e = E() }; // #dr977-E #if !defined(_WIN32) || defined(__MINGW32__) -// expected-error@-2 {{invalid use of incomplete type 'E'}} -// expected-note@-3 {{definition of 'dr977::E' is not complete until the closing '}'}} +// expected-error@#dr977-E {{invalid use of incomplete type 'E'}} +// expected-note@#dr977-E {{definition of 'dr977::E' is not complete until the closing '}'}} #endif #if __cplusplus >= 201103L enum E2 : int { e2 = E2() }; @@ -105,23 +109,29 @@ enum struct E4 : int { e = static_cast(E4()) }; namespace dr990 { // dr990: 3.5 #if __cplusplus >= 201103L - struct A { // expected-note 2{{candidate}} - A(std::initializer_list); // expected-note {{candidate}} + struct A { // #dr990-A + A(std::initializer_list); // #dr990-A-init-list }; struct B { A a; }; B b1 { }; - B b2 { 1 }; // expected-error {{no viable conversion from 'int' to 'A'}} + B b2 { 1 }; + // since-cxx11-error@-1 {{no viable conversion from 'int' to 'A'}} + // since-cxx11-note@#dr990-A {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'int' to 'const A &' for 1st argument}} + // since-cxx11-note@#dr990-A {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'int' to 'A &&' for 1st argument}} + // since-cxx11-note@#dr990-A-init-list {{candidate constructor not viable: no known conversion from 'int' to 'std::initializer_list' for 1st argument}} B b3 { { 1 } }; struct C { C(); C(int); - C(std::initializer_list) = delete; // expected-note {{here}} + C(std::initializer_list) = delete; // #dr990-deleted }; C c1[3] { 1 }; // ok - C c2[3] { 1, {2} }; // expected-error {{call to deleted}} + C c2[3] { 1, {2} }; + // since-cxx11-error@-1 {{call to deleted constructor of 'C'}} + // since-cxx11-note@#dr990-deleted {{'C' has been explicitly marked deleted here}} struct D { D(); diff --git a/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp b/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp index ab1b9f7a73eec8912173d953fc5756473bc47e8a..1cf9e1c9f9c0fa39caf6c1d326e68519cee41a62 100644 --- a/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp +++ b/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp @@ -17,7 +17,7 @@ public: for (U count = n.count_; count; --count) x += a; return x; - } + } }; friend Num operator+(const Num &a, const Num &b) { @@ -145,7 +145,7 @@ namespace test5 { namespace Dependent { template class X; - template + template X operator+(const X&, const T*); template class X { @@ -249,7 +249,7 @@ namespace test11 { }; template struct Foo::IteratorImpl; - template struct Foo::IteratorImpl; + template struct Foo::IteratorImpl; } // PR6827 diff --git a/clang/test/CodeGen/RISCV/riscv-func-attr-target.c b/clang/test/CodeGen/RISCV/riscv-func-attr-target.c index 74bc5f2ac70492ed13fe7861a03d01d572f02002..506acaba687417ab475ec748b6bfcdb6764e26a0 100644 --- a/clang/test/CodeGen/RISCV/riscv-func-attr-target.c +++ b/clang/test/CodeGen/RISCV/riscv-func-attr-target.c @@ -1,6 +1,7 @@ // REQUIRES: riscv-registered-target // RUN: %clang_cc1 -triple riscv64 -target-feature +zifencei -target-feature +m \ -// RUN: -target-feature +a -target-feature +save-restore \ +// RUN: -target-feature +a -target-feature +save-restore -target-feature -zbb \ +// RUN: -target-feature -relax -target-feature -zfa \ // RUN: -emit-llvm %s -o - | FileCheck %s // CHECK-LABEL: define dso_local void @testDefault @@ -35,12 +36,12 @@ testAttrFullArchAndAttrCpu() {} __attribute__((target("cpu=sifive-u54"))) void testAttrCpuOnly() {} //. -// CHECK: attributes #0 = { {{.*}}"target-features"="+64bit,+a,+m,+save-restore,+zifencei" } -// CHECK: attributes #1 = { {{.*}}"target-cpu"="rocket-rv64" "target-features"="+64bit,+a,+d,+f,+m,+save-restore,+v,+zicsr,+zifencei,+zve32f,+zve32x,+zve64d,+zve64f,+zve64x,+zvl128b,+zvl32b,+zvl64b" "tune-cpu"="generic-rv64" } -// CHECK: attributes #2 = { {{.*}}"target-features"="+64bit,+a,+m,+save-restore,+zbb,+zifencei" } -// CHECK: attributes #3 = { {{.*}}"target-features"="+64bit,+a,+d,+experimental-zicond,+f,+m,+save-restore,+v,+zbb,+zicsr,+zifencei,+zve32f,+zve32x,+zve64d,+zve64f,+zve64x,+zvl128b,+zvl32b,+zvl64b" } -// CHECK: attributes #4 = { {{.*}}"target-features"="+64bit,+a,+c,+d,+f,+m,+save-restore,+zbb,+zicsr,+zifencei" } -// CHECK: attributes #5 = { {{.*}}"target-features"="+64bit,+m,+save-restore" } -// CHECK: attributes #6 = { {{.*}}"target-cpu"="sifive-u54" "target-features"="+64bit,+a,+m,+save-restore,+zbb,+zifencei" } -// CHECK: attributes #7 = { {{.*}}"target-cpu"="sifive-u54" "target-features"="+64bit,+m,+save-restore" } -// CHECK: attributes #8 = { {{.*}}"target-cpu"="sifive-u54" "target-features"="+64bit,+a,+c,+d,+f,+m,+save-restore,+zicsr,+zifencei" } +// CHECK: attributes #0 = { {{.*}}"target-features"="+64bit,+a,+m,+save-restore,+zifencei,-relax,-zbb,-zfa" } +// CHECK: attributes #1 = { {{.*}}"target-cpu"="rocket-rv64" "target-features"="+64bit,+a,+d,+f,+m,+save-restore,+v,+zicsr,+zifencei,+zve32f,+zve32x,+zve64d,+zve64f,+zve64x,+zvl128b,+zvl32b,+zvl64b,-relax,-zbb,-zfa" "tune-cpu"="generic-rv64" } +// CHECK: attributes #2 = { {{.*}}"target-features"="+64bit,+a,+m,+save-restore,+zbb,+zifencei,-relax,-zfa" } +// CHECK: attributes #3 = { {{.*}}"target-features"="+64bit,+a,+d,+experimental-zicond,+f,+m,+save-restore,+v,+zbb,+zicsr,+zifencei,+zve32f,+zve32x,+zve64d,+zve64f,+zve64x,+zvl128b,+zvl32b,+zvl64b,-relax,-zfa" } +// CHECK: attributes #4 = { {{.*}}"target-features"="+64bit,+a,+c,+d,+f,+m,+save-restore,+zbb,+zicsr,+zifencei,-relax,-zfa" } +// CHECK: attributes #5 = { {{.*}}"target-features"="+64bit,+m,+save-restore,-relax,-zbb,-zfa" } +// CHECK: attributes #6 = { {{.*}}"target-cpu"="sifive-u54" "target-features"="+64bit,+a,+m,+save-restore,+zbb,+zifencei,-relax,-zfa" } +// CHECK: attributes #7 = { {{.*}}"target-cpu"="sifive-u54" "target-features"="+64bit,+m,+save-restore,-relax,-zbb,-zfa" } +// CHECK: attributes #8 = { {{.*}}"target-cpu"="sifive-u54" "target-features"="+64bit,+a,+c,+d,+f,+m,+save-restore,+zicsr,+zifencei,-relax,-zbb,-zfa" } diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/rvv-error.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/rvv-error.c index 1a29acbf3ba92d027e40c43f8c14b6d5618126da..6ec9b057997690a919a39da38a85e31372af0ffe 100644 --- a/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/rvv-error.c +++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/rvv-error.c @@ -11,7 +11,7 @@ // CHECK-RV64V-NEXT: ret i32 [[CONV]] // -// CHECK-RV64-ERR: error: builtin requires at least one of the following extensions to be enabled: 'Zve32x' +// CHECK-RV64-ERR: error: builtin requires at least one of the following extensions: 'Zve32x' int test() { return __builtin_rvv_vsetvli(1, 0, 0); diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c new file mode 100644 index 0000000000000000000000000000000000000000..82b30e8bbe9b93a0f7ddb71576f6ebe0f65df338 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c @@ -0,0 +1,47 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py + +// REQUIRES: aarch64-registered-target + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin.§ +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +// CHECK-LABEL: @test_svreinterpret_svbool_svcnt( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[CNT:%.*]]) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z31test_svreinterpret_svbool_svcntu11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[CNT:%.*]]) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svbool_t test_svreinterpret_svbool_svcnt(svcount_t cnt) __arm_streaming_compatible +{ + return SVE_ACLE_FUNC(svreinterpret,_b,,)(cnt); +} + +// CHECK-LABEL: @test_svreinterpret_svcnt_svbool( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[PG:%.*]]) +// CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z31test_svreinterpret_svcnt_svboolu10__SVBool_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[PG:%.*]]) +// CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] +// +svcount_t test_svreinterpret_svcnt_svbool(svbool_t pg) __arm_streaming_compatible +{ + return SVE_ACLE_FUNC(svreinterpret,_c,,)(pg); +} diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_qrshr.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_qrshr.c new file mode 100644 index 0000000000000000000000000000000000000000..50cac4888789461261e627581164af3aaeb6ff12 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_qrshr.c @@ -0,0 +1,343 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: aarch64-registered-target + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED,A5) A1##A3##A5 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4,A5) A1##A2##A3##A4##A5 +#endif + +// SVQRSHR + +// CHECK-LABEL: @test_svsqrshr_u16_u32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.uqrshr.x2.nxv4i32( [[TMP0]], [[TMP1]], i32 16) +// CHECK-NEXT: ret [[TMP2]] +// +// CPP-CHECK-LABEL: @_Z24test_svsqrshr_u16_u32_x412svuint32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.uqrshr.x2.nxv4i32( [[TMP0]], [[TMP1]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP2]] +// +svuint16_t test_svsqrshr_u16_u32_x4(svuint32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshr,_n,_u16,_u32_x2,)(zn, 16); +} + +// CHECK-LABEL: @test_svsqrshr_s16_s32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.sqrshr.x2.nxv4i32( [[TMP0]], [[TMP1]], i32 16) +// CHECK-NEXT: ret [[TMP2]] +// +// CPP-CHECK-LABEL: @_Z24test_svsqrshr_s16_s32_x411svint32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.sqrshr.x2.nxv4i32( [[TMP0]], [[TMP1]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP2]] +// +svint16_t test_svsqrshr_s16_s32_x4(svint32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshr,_n,_s16,_s32_x2,)(zn, 16); +} + +// CHECK-LABEL: @test_svsqrshr_u8_u32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshr.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z23test_svsqrshr_u8_u32_x412svuint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshr.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint8_t test_svsqrshr_u8_u32_x4(svuint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshr,_n,_u8,_u32_x4,)(zn, 8); +} + +// CHECK-LABEL: @test_svsqrshr_s8_s32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshr.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z23test_svsqrshr_s8_s32_x411svint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshr.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svint8_t test_svsqrshr_s8_s32_x4(svint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshr,_n,_s8,_s32_x4,)(zn, 8); +} + +// CHECK-LABEL: @test_svsqrshr_u16_u64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshr.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z24test_svsqrshr_u16_u64_x412svuint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshr.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint16_t test_svsqrshr_u16_u64_x4(svuint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshr,_n,_u16,_u64_x4,)(zn, 16); +} + +// CHECK-LABEL: @test_svsqrshr_s16_s64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshr.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z24test_svsqrshr_s16_s64_x411svint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshr.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svint16_t test_svsqrshr_s16_s64_x4(svint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshr,_n,_s16,_s64_x4,)(zn, 16); +} + +// SVQRSHRN + +// CHECK-LABEL: @test_svsqrshrn_u8_u32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshrn.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z24test_svsqrshrn_u8_u32_x412svuint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshrn.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint8_t test_svsqrshrn_u8_u32_x4(svuint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshrn,_n,_u8,_u32_x4,)(zn, 8); +} + +// CHECK-LABEL: @test_svsqrshrn_s8_s32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrn.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z24test_svsqrshrn_s8_s32_x411svint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrn.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svint8_t test_svsqrshrn_s8_s32_x4(svint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshrn,_n,_s8,_s32_x4,)(zn, 8); +} + +// CHECK-LABEL: @test_svsqrshrn_u16_u64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshrn.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z25test_svsqrshrn_u16_u64_x412svuint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshrn.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint16_t test_svsqrshrn_u16_u64_x4(svuint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshrn,_n,_u16,_u64_x4,)(zn, 16); +} + +// CHECK-LABEL: @test_svsqrshrn_s16_s64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrn.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z25test_svsqrshrn_s16_s64_x411svint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrn.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svint16_t test_svsqrshrn_s16_s64_x4(svint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshrn,_n,_s16,_s64_x4,)(zn, 16); +} + +// SVSQRSHRU + +// CHECK-LABEL: @test_svsvqrshru_u16_s32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.sqrshru.x2.nxv4i32( [[TMP0]], [[TMP1]], i32 16) +// CHECK-NEXT: ret [[TMP2]] +// +// CPP-CHECK-LABEL: @_Z26test_svsvqrshru_u16_s32_x211svint32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.sqrshru.x2.nxv4i32( [[TMP0]], [[TMP1]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP2]] +// +svuint16_t test_svsvqrshru_u16_s32_x2(svint32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshru,_n,_u16,_s32_x2,)(zn, 16); +} + +// CHECK-LABEL: @test_svsqrshru_u8_s32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshru.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z24test_svsqrshru_u8_s32_x411svint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshru.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint8_t test_svsqrshru_u8_s32_x4(svint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshru,_n,_u8,_s32_x4,)(zn, 8); +} + +// CHECK-LABEL: @test_svsqrshru_u16_s64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshru.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z25test_svsqrshru_u16_s64_x411svint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshru.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint16_t test_svsqrshru_u16_s64_x4(svint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshru,_n,_u16,_s64_x4,)(zn, 16); +} + +// SQRSHRUN x 4 + +// CHECK-LABEL: @test_svsqrshrun_u8_s32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrun.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 32) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z25test_svsqrshrun_u8_s32_x411svint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrun.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 32) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint8_t test_svsqrshrun_u8_s32_x4(svint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshrun,_n,_u8,_s32_x4,)(zn, 32); +} + +// CHECK-LABEL: @test_svsqrshrun_u16_s64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrun.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 64) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z26test_svsqrshrun_u16_s64_x411svint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrun.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 64) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint16_t test_svsqrshrun_u16_s64_x4(svint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshrun,_n,_u16,_s64_x4,)(zn, 64); +} diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create2_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create2_bool.c new file mode 100644 index 0000000000000000000000000000000000000000..eb5a19b1d9d326e1a8c3f578a731bbbeee40da82 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create2_bool.c @@ -0,0 +1,38 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s\ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +// REQUIRES: aarch64-registered-target + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +// CHECK-LABEL: @test_svcreate2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[X0:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP0]], [[X1:%.*]], i64 16) +// CHECK-NEXT: ret [[TMP1]] +// +// CPP-CHECK-LABEL: @_Z17test_svcreate2_s8u10__SVBool_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[X0:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP0]], [[X1:%.*]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP1]] +// +svboolx2_t test_svcreate2_s8(svbool_t x0, svbool_t x1) +{ + return SVE_ACLE_FUNC(svcreate2,_b8,,)(x0, x1); +} diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create4_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create4_bool.c new file mode 100644 index 0000000000000000000000000000000000000000..3f99ee6e79c0769efe3fa22918bc123cc72b7442 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create4_bool.c @@ -0,0 +1,42 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s\ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +// REQUIRES: aarch64-registered-target + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +// CHECK-LABEL: @test_svcreate4_b8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( poison, [[X0:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TMP0]], [[X1:%.*]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TMP1]], [[X2:%.*]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TMP2]], [[X4:%.*]], i64 48) +// CHECK-NEXT: ret [[TMP3]] +// +// CPP-CHECK-LABEL: @_Z17test_svcreate4_b8u10__SVBool_tS_S_S_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( poison, [[X0:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TMP0]], [[X1:%.*]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TMP1]], [[X2:%.*]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TMP2]], [[X4:%.*]], i64 48) +// CPP-CHECK-NEXT: ret [[TMP3]] +// +svboolx4_t test_svcreate4_b8(svbool_t x0, svbool_t x1, svbool_t x2, svbool_t x4) +{ + return SVE_ACLE_FUNC(svcreate4,_b8,,)(x0, x1, x2, x4); +} diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get2_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get2_bool.c new file mode 100644 index 0000000000000000000000000000000000000000..5d38f72b34b164c7b2ebfbd8cd7dfe81ec6510db --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get2_bool.c @@ -0,0 +1,49 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s\ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// REQUIRES: aarch64-registered-target +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +// CHECK-LABEL: @test_svget2_b8_0( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv32i1( [[TUPLE:%.*]], i64 0) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svget2_b8_010svboolx2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv32i1( [[TUPLE:%.*]], i64 0) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svbool_t test_svget2_b8_0(svboolx2_t tuple) +{ + return SVE_ACLE_FUNC(svget2,_b8,,)(tuple, 0); +} + +// CHECK-LABEL: @test_svget2_b8_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv32i1( [[TUPLE:%.*]], i64 16) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svget2_b8_110svboolx2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv32i1( [[TUPLE:%.*]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svbool_t test_svget2_b8_1(svboolx2_t tuple) +{ + return SVE_ACLE_FUNC(svget2,_b8,,)(tuple, 1); +} diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get4_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get4_bool.c new file mode 100644 index 0000000000000000000000000000000000000000..821a5be3254ebad4fafc8c3bd532783568d574dd --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get4_bool.c @@ -0,0 +1,72 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s\ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +// REQUIRES: aarch64-registered-target + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +// NOTE: For these tests clang converts the struct parameter into +// several parameters, one for each member of the original struct. +// CHECK-LABEL: @test_svget4_b8_0( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv64i1( [[TUPLE:%.*]], i64 0) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svget4_b8_010svboolx4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv64i1( [[TUPLE:%.*]], i64 0) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svbool_t test_svget4_b8_0(svboolx4_t tuple) +{ + return SVE_ACLE_FUNC(svget4,_b8,,)(tuple, 0); +} + +// NOTE: For these tests clang converts the struct parameter into +// several parameters, one for each member of the original struct. +// CHECK-LABEL: @test_svget4_b8_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv64i1( [[TUPLE:%.*]], i64 16) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svget4_b8_110svboolx4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv64i1( [[TUPLE:%.*]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svbool_t test_svget4_b8_1(svboolx4_t tuple) +{ + return SVE_ACLE_FUNC(svget4,_b8,,)(tuple, 1); +} + +// NOTE: For these tests clang converts the struct parameter into +// several parameters, one for each member of the original struct. +// CHECK-LABEL: @test_svget4_b8_3( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv64i1( [[TUPLE:%.*]], i64 48) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svget4_b8_310svboolx4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv64i1( [[TUPLE:%.*]], i64 48) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svbool_t test_svget4_b8_3(svboolx4_t tuple) +{ + return SVE_ACLE_FUNC(svget4,_b8,,)(tuple, 3); +} diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c index fe15d5a9db81f2f0f8956751908696e873cd9a44..a3206029019c3dd1da3b206b4aee4efd0e7af9c4 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c @@ -1,10 +1,17 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -DTEST_SME2 -target-feature +sve -target-feature +sme2 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + // CHECK-LABEL: @test_svpext_lane_c8_0( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 0) @@ -15,7 +22,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 0) // CPP-CHECK-NEXT: ret [[TMP0]] // -svbool_t test_svpext_lane_c8_0(svcount_t c) { +svbool_t test_svpext_lane_c8_0(svcount_t c) ATTR { return svpext_lane_c8(c, 0); } @@ -29,7 +36,7 @@ svbool_t test_svpext_lane_c8_0(svcount_t c) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 3) // CPP-CHECK-NEXT: ret [[TMP0]] // -svbool_t test_svpext_lane_c8_3(svcount_t c) { +svbool_t test_svpext_lane_c8_3(svcount_t c) ATTR { return svpext_lane_c8(c, 3); } @@ -45,7 +52,7 @@ svbool_t test_svpext_lane_c8_3(svcount_t c) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP0]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpext_lane_c16_0(svcount_t c) { +svbool_t test_svpext_lane_c16_0(svcount_t c) ATTR { return svpext_lane_c16(c, 0); } @@ -61,7 +68,7 @@ svbool_t test_svpext_lane_c16_0(svcount_t c) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP0]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpext_lane_c16_3(svcount_t c) { +svbool_t test_svpext_lane_c16_3(svcount_t c) ATTR { return svpext_lane_c16(c, 3); } @@ -77,7 +84,7 @@ svbool_t test_svpext_lane_c16_3(svcount_t c) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP0]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpext_lane_c32_0(svcount_t c) { +svbool_t test_svpext_lane_c32_0(svcount_t c) ATTR { return svpext_lane_c32(c, 0); } @@ -93,7 +100,7 @@ svbool_t test_svpext_lane_c32_0(svcount_t c) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP0]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpext_lane_c32_3(svcount_t c) { +svbool_t test_svpext_lane_c32_3(svcount_t c) ATTR { return svpext_lane_c32(c, 3); } @@ -109,7 +116,7 @@ svbool_t test_svpext_lane_c32_3(svcount_t c) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP0]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpext_lane_c64_0(svcount_t c) { +svbool_t test_svpext_lane_c64_0(svcount_t c) ATTR { return svpext_lane_c64(c, 0); } @@ -125,7 +132,7 @@ svbool_t test_svpext_lane_c64_0(svcount_t c) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP0]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpext_lane_c64_3(svcount_t c) { +svbool_t test_svpext_lane_c64_3(svcount_t c) ATTR { return svpext_lane_c64(c, 3); } @@ -147,6 +154,184 @@ svbool_t test_svpext_lane_c64_3(svcount_t c) { // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) // CPP-CHECK-NEXT: ret [[TMP4]] // -svboolx2_t test_svpext_lane_c8_x2_0(svcount_t c) { +svboolx2_t test_svpext_lane_c8_x2_0(svcount_t c) ATTR { return svpext_lane_c8_x2(c, 0); } + +// CHECK-LABEL: @test_svpext_lane_c8_x2_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z24test_svpext_lane_c8_x2_1u11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svboolx2_t test_svpext_lane_c8_x2_1(svcount_t c) ATTR { + return svpext_lane_c8_x2(c, 1); +} + +// CHECK-LABEL: @test_svpext_lane_c16_x2_0( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv8i1(target("aarch64.svcount") [[C:%.*]], i32 0) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z25test_svpext_lane_c16_x2_0u11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv8i1(target("aarch64.svcount") [[C:%.*]], i32 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svpext_lane_c16_x2_0(svcount_t c) ATTR { + return svpext_lane_c16_x2(c, 0); +} + +// CHECK-LABEL: @test_svpext_lane_c16_x2_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv8i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z25test_svpext_lane_c16_x2_1u11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv8i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svpext_lane_c16_x2_1(svcount_t c) ATTR { + return svpext_lane_c16_x2(c, 1); +} + +// CHECK-LABEL: @test_svpext_lane_c32_x2_0( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv4i1(target("aarch64.svcount") [[C:%.*]], i32 0) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z25test_svpext_lane_c32_x2_0u11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv4i1(target("aarch64.svcount") [[C:%.*]], i32 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svpext_lane_c32_x2_0(svcount_t c) ATTR { + return svpext_lane_c32_x2(c, 0); +} + +// CHECK-LABEL: @test_svpext_lane_c32_x2_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv4i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z25test_svpext_lane_c32_x2_1u11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv4i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svpext_lane_c32_x2_1(svcount_t c) ATTR { + return svpext_lane_c32_x2(c, 1); +} + +// CHECK-LABEL: @test_svpext_lane_c64_x2_0( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv2i1(target("aarch64.svcount") [[C:%.*]], i32 0) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z25test_svpext_lane_c64_x2_0u11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv2i1(target("aarch64.svcount") [[C:%.*]], i32 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svpext_lane_c64_x2_0(svcount_t c) ATTR { + return svpext_lane_c64_x2(c, 0); +} + +// CHECK-LABEL: @test_svpext_lane_c64_x2_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv2i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z25test_svpext_lane_c64_x2_1u11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv2i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svpext_lane_c64_x2_1(svcount_t c) ATTR { + return svpext_lane_c64_x2(c, 1); +} diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c index aa2a35c2fd2541c7986333ee1804916d5cfe5fe8..73b7b0347dd970974824308600c855da81212cd9 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c @@ -5,6 +5,11 @@ // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ // RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ +// RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ +// RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s #include @@ -20,7 +25,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.psel.nxv16i1( [[P1:%.*]], [[P2:%.*]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) { +svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_b8(p1, p2, idx + 15); } @@ -38,7 +43,7 @@ svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv8i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) { +svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_b16(p1, p2, idx + 7); } @@ -56,7 +61,7 @@ svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv4i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) { +svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_b32(p1, p2, idx + 3); } @@ -74,7 +79,7 @@ svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv2i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) { +svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_b64(p1, p2, idx + 1); } @@ -94,7 +99,7 @@ svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) { // CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP1]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP2]] // -svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) { +svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_c8(p1, p2, idx + 15); } @@ -116,7 +121,7 @@ svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) { // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) { +svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_c16(p1, p2, idx + 7); } @@ -138,7 +143,7 @@ svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) { // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) { +svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_c32(p1, p2, idx + 3); } @@ -160,6 +165,6 @@ svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) { // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c64(svcount_t p1, svbool_t p2, uint32_t idx) { +svcount_t test_svpsel_lane_c64(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_c64(p1, p2, idx + 1); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_set2_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_set2_bool.c new file mode 100644 index 0000000000000000000000000000000000000000..298e5b80ceb004b14dc6b3b81fb5b732ef1fdabb --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_set2_bool.c @@ -0,0 +1,52 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s\ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +// REQUIRES: aarch64-registered-target + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +// CHECK-LABEL: @test_svset2_b8_0( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 0) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svset2_b8_010svboolx2_tu10__SVBool_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 0) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svboolx2_t test_svset2_b8_0(svboolx2_t tuple, svbool_t x) +{ + return SVE_ACLE_FUNC(svset2,_b8,,)(tuple, 0, x); +} + +// CHECK-LABEL: @test_svset2_b8_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 16) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svset2_b8_110svboolx2_tu10__SVBool_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svboolx2_t test_svset2_b8_1(svboolx2_t tuple, svbool_t x) +{ + return SVE_ACLE_FUNC(svset2,_b8,,)(tuple, 1, x); +} + diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_set4_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_set4_bool.c new file mode 100644 index 0000000000000000000000000000000000000000..746a8280ac53f3036b5de2fa1f72b4115054c114 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_set4_bool.c @@ -0,0 +1,66 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s\ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// REQUIRES: aarch64-registered-target + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + + +// CHECK-LABEL: @test_svset4_b8_0( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 0) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svset4_b8_010svboolx4_tu10__SVBool_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 0) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svboolx4_t test_svset4_b8_0(svboolx4_t tuple, svbool_t x) +{ + return SVE_ACLE_FUNC(svset4,_b8,,)(tuple, 0, x); +} + +// CHECK-LABEL: @test_svset4_b8_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 16) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svset4_b8_110svboolx4_tu10__SVBool_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svboolx4_t test_svset4_b8_1(svboolx4_t tuple, svbool_t x) +{ + return SVE_ACLE_FUNC(svset4,_b8,,)(tuple, 1, x); +} + +// CHECK-LABEL: @test_svset4_b8_3( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 48) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svset4_b8_310svboolx4_tu10__SVBool_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 48) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svboolx4_t test_svset4_b8_3(svboolx4_t tuple, svbool_t x) +{ + return SVE_ACLE_FUNC(svset4,_b8,,)(tuple, 3, x); +} diff --git a/clang/test/CodeGen/arm-target-features.c b/clang/test/CodeGen/arm-target-features.c index bd051059fb0cc178375cf76520e877b01c2474e1..ad4bfd45c408b5481060f09937a4a593502aab31 100644 --- a/clang/test/CodeGen/arm-target-features.c +++ b/clang/test/CodeGen/arm-target-features.c @@ -113,6 +113,9 @@ // RUN: %clang_cc1 -triple thumb-linux-gnueabi -target-cpu cortex-m85 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-ARMV81M-CORTEX-M85-LINUX // CHECK-ARMV81M-CORTEX-M85-LINUX: "target-features"="+armv8.1-m.main,+dsp,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fullfp16,+hwdiv,+lob,+mve,+mve.fp,+pacbti,+ras,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp" +// RUN: %clang_cc1 -triple thumb-linux-gnueabi -target-cpu cortex-m52 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-ARMV81M-CORTEX-M52-LINUX +// CHECK-ARMV81M-CORTEX-M52-LINUX: "target-features"="+armv8.1-m.main,+dsp,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fullfp16,+hwdiv,+lob,+mve,+mve.fp,+pacbti,+ras,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp" + // RUN: %clang_cc1 -triple thumbv9.3a-linux-gnueabihf -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-ARCH93 // CHECK-ARCH93: "target-features"="+armv9.3-a,+thumb-mode,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8.7a,+v8.8a,+v9.1a,+v9.2a,+v9.3a,+v9a" diff --git a/clang/test/CodeGen/complex-math.c b/clang/test/CodeGen/complex-math.c index c59baaa452369d66d71362b71e4efaafe77ca240..a44aa0014a6587955bdac9d6fca37d5e42adb3d7 100644 --- a/clang/test/CodeGen/complex-math.c +++ b/clang/test/CodeGen/complex-math.c @@ -5,7 +5,7 @@ // RUN: %clang_cc1 %s -O0 -emit-llvm -triple armv7-none-linux-gnueabi -o - | FileCheck %s --check-prefix=ARM // RUN: %clang_cc1 %s -O0 -emit-llvm -triple armv7-none-linux-gnueabihf -o - | FileCheck %s --check-prefix=ARMHF // RUN: %clang_cc1 %s -O0 -emit-llvm -triple thumbv7k-apple-watchos2.0 -o - -target-abi aapcs16 | FileCheck %s --check-prefix=ARM7K -// RUN: %clang_cc1 %s -O0 -emit-llvm -triple aarch64-unknown-unknown -ffast-math -ffp-contract=fast -o - | FileCheck %s --check-prefix=AARCH64-FASTMATH +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple aarch64-unknown-unknown -ffast-math -ffp-contract=fast -complex-range=fortran -o - | FileCheck %s --check-prefix=AARCH64-FASTMATH // RUN: %clang_cc1 %s -O0 -emit-llvm -triple spir -o - | FileCheck %s --check-prefix=SPIR float _Complex add_float_rr(float a, float b) { @@ -135,24 +135,68 @@ float _Complex div_float_rc(float a, float _Complex b) { // SPIR: call spir_func {{.*}} @__divsc3( - // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD)) + // a / b = (A+iB) / (C+iD) = (E+iF) + // if (|C| >= |D|) + // DdC = D/C + // CpRD = C+DdC*D + // E = (A+B*DdC)/CpRD + // F = (B-A*DdC)/CpRD + // else + // CdD = C/D + // DpRC= D+CdD*C + // E = (A*CdD+B)/DpRC + // F = (B*CdD-A)/DpRC // AARCH64-FASTMATH-LABEL: @div_float_rc(float noundef nofpclass(nan inf) %a, [2 x float] noundef nofpclass(nan inf) alignstack(8) %b.coerce) - // A = a - // B = 0 - // - // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast float - // BD = 0 - // ACpBD = AC - // - // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast float - // - // BC = 0 - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast float - // - // AARCH64-FASTMATH: fdiv fast float - // AARCH64-FASTMATH: fdiv fast float + // |C| + // AARCH64-FASTMATH: call {{.*}}float @llvm.fabs.f32(float {{.*}}) + // |D| + // AARCH64-FASTMATH-NEXT: call {{.*}}float @llvm.fabs.f32(float {{.*}}) + // AARCH64-FASTMATH-NEXT: fcmp {{.*}}ugt float + // AARCH64-FASTMATH-NEXT: br i1 {{.*}}, label + // AARCH64-FASTMATH: abs_rhsr_greater_or_equal_abs_rhsi: + + // |C| >= |D| + // DdC=D/C + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // CpRD=C+CdC*D + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + + // A+BR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // B-AR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fsub {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: abs_rhsr_less_than_abs_rhsi: + + // |C| < |D| + // CdD=C/D + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // DpRC=D+CdD*C + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + + // (A*CdD+B)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // (BCdD-A)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fsub {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: complex_div: + // AARCH64-FASTMATH-NEXT: phi {{.*}}float + // AARCH64-FASTMATH-NEXT: phi {{.*}}float // AARCH64-FASTMATH: ret return a / b; } @@ -164,24 +208,68 @@ float _Complex div_float_cc(float _Complex a, float _Complex b) { // SPIR: call spir_func {{.*}} @__divsc3( - // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD)) + // a / b = (A+iB) / (C+iD) = (E+iF) + // if (|C| >= |D|) + // DdC = D/C + // CpRD = C+DdC*D + // E = (A+B*DdC)/CpRD + // F = (B-A*DdC)/CpRD + // else + // CdD = C/D + // DpRC= D+CdD*C + // E = (A*CdD+B)/DpRC + // F = (B*CdD-A)/DpRC // AARCH64-FASTMATH-LABEL: @div_float_cc([2 x float] noundef nofpclass(nan inf) alignstack(8) %a.coerce, [2 x float] noundef nofpclass(nan inf) alignstack(8) %b.coerce) - // - // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[BD:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[ACpBD:%.*]] = fadd fast float - // - // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast float - // - // AARCH64-FASTMATH: [[BC:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[BCmAD:%.*]] = fsub fast float - // - // AARCH64-FASTMATH: fdiv fast float - // AARCH64-FASTMATH: fdiv fast float - // AARCH64-FASTMATH: ret + // |C| + // AARCH64-FASTMATH: call {{.*}}float @llvm.fabs.f32(float {{.*}}) + // |D| + // AARCH64-FASTMATH-NEXT: call {{.*}}float @llvm.fabs.f32(float {{.*}}) + // AARCH64-FASTMATH-NEXT: fcmp {{.*}}ugt float + // AARCH64-FASTMATH-NEXT: br i1 {{.*}}, label + // AARCH64-FASTMATH: abs_rhsr_greater_or_equal_abs_rhsi: + + // |C| >= |D| + // DdC=D/C + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // CpRD=C+CdC*D + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + + // A+BR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // B-AR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fsub {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: abs_rhsr_less_than_abs_rhsi: + + // |C| < |D| + // CdD=C/D + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // DpRC=D+CdD*C + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + + // (A*CdD+B)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // (BCdD-A)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fsub {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: complex_div: + // AARCH64-FASTMATH-NEXT: phi {{.*}}float + // AARCH64-FASTMATH-NEXT: phi {{.*}}float return a / b; } @@ -312,24 +400,68 @@ double _Complex div_double_rc(double a, double _Complex b) { // SPIR: call spir_func {{.*}} @__divdc3( - // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD)) + // a / b = (A+iB) / (C+iD) = (E+iF) + // if (|C| >= |D|) + // DdC = D/C + // CpRD = C+DdC*D + // E = (A+B*DdC)/CpRD + // F = (B-A*DdC)/CpRD + // else + // CdD = C/D + // DpRC= D+CdD*C + // E = (A*CdD+B)/DpRC + // F = (B*CdD-A)/DpRC // AARCH64-FASTMATH-LABEL: @div_double_rc(double noundef nofpclass(nan inf) %a, [2 x double] noundef nofpclass(nan inf) alignstack(8) %b.coerce) - // A = a - // B = 0 - // - // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast double - // BD = 0 - // ACpBD = AC - // - // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast double - // - // BC = 0 - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast double - // - // AARCH64-FASTMATH: fdiv fast double - // AARCH64-FASTMATH: fdiv fast double + // |C| + // AARCH64-FASTMATH: call {{.*}}double @llvm.fabs.f64(double {{.*}}) + // |D| + // AARCH64-FASTMATH-NEXT: call {{.*}}double @llvm.fabs.f64(double {{.*}}) + // AARCH64-FASTMATH-NEXT: fcmp {{.*}}ugt double + // AARCH64-FASTMATH-NEXT: br i1 {{.*}}, label + // AARCH64-FASTMATH: abs_rhsr_greater_or_equal_abs_rhsi: + + // |C| >= |D| + // DdC=D/C + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // CpRD=C+CdC*D + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + + // A+BR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // B-AR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fsub {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: abs_rhsr_less_than_abs_rhsi: + + // |C| < |D| + // CdD=C/D + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // DpRC=D+CdD*C + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + + // (A*CdD+B)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // (BCdD-A)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fsub {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: complex_div: + // AARCH64-FASTMATH-NEXT: phi {{.*}}double + // AARCH64-FASTMATH-NEXT: phi {{.*}}double // AARCH64-FASTMATH: ret return a / b; } @@ -341,23 +473,68 @@ double _Complex div_double_cc(double _Complex a, double _Complex b) { // SPIR: call spir_func {{.*}} @__divdc3( - // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD)) + // a / b = (A+iB) / (C+iD) = (E+iF) + // if (|C| >= |D|) + // DdC = D/C + // CpRD = C+DdC*D + // E = (A+B*DdC)/CpRD + // F = (B-A*DdC)/CpRD + // else + // CdD = C/D + // DpRC= D+CdD*C + // E = (A*CdD+B)/DpRC + // F = (B*CdD-A)/DpRC // AARCH64-FASTMATH-LABEL: @div_double_cc([2 x double] noundef nofpclass(nan inf) alignstack(8) %a.coerce, [2 x double] noundef nofpclass(nan inf) alignstack(8) %b.coerce) - // - // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[BD:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[ACpBD:%.*]] = fadd fast double - // - // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast double - // - // AARCH64-FASTMATH: [[BC:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[BCmAD:%.*]] = fsub fast double - // - // AARCH64-FASTMATH: fdiv fast double - // AARCH64-FASTMATH: fdiv fast double + // |C| + // AARCH64-FASTMATH: call {{.*}}double @llvm.fabs.f64(double {{.*}}) + // |D| + // AARCH64-FASTMATH-NEXT: call {{.*}}double @llvm.fabs.f64(double {{.*}}) + // AARCH64-FASTMATH-NEXT: fcmp {{.*}}ugt double + // AARCH64-FASTMATH-NEXT: br i1 {{.*}}, label + // AARCH64-FASTMATH: abs_rhsr_greater_or_equal_abs_rhsi: + + // |C| >= |D| + // DdC=D/C + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // CpRD=C+CdC*D + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + + // A+BR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // B-AR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fsub {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: abs_rhsr_less_than_abs_rhsi: + + // |C| < |D| + // CdD=C/D + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // DpRC=D+CdD*C + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + + // (A*CdD+B)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // (BCdD-A)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fsub {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: complex_div: + // AARCH64-FASTMATH-NEXT: phi {{.*}}double + // AARCH64-FASTMATH-NEXT: phi {{.*}}double // AARCH64-FASTMATH: ret return a / b; } @@ -505,24 +682,68 @@ long double _Complex div_long_double_rc(long double a, long double _Complex b) { // PPC: ret // SPIR: call spir_func {{.*}} @__divdc3( - // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD)) + // a / b = (A+iB) / (C+iD) = (E+iF) + // if (|C| >= |D|) + // DdC = D/C + // CpRD = C+DdC*D + // E = (A+B*DdC)/CpRD + // F = (B-A*DdC)/CpRD + // else + // CdD = C/D + // DpRC= D+CdD*C + // E = (A*CdD+B)/DpRC + // F = (B*CdD-A)/DpRC // AARCH64-FASTMATH-LABEL: @div_long_double_rc(fp128 noundef nofpclass(nan inf) %a, [2 x fp128] noundef nofpclass(nan inf) alignstack(16) %b.coerce) - // A = a - // B = 0 - // - // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast fp128 - // BD = 0 - // ACpBD = AC - // - // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast fp128 - // - // BC = 0 - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast fp128 - // - // AARCH64-FASTMATH: fdiv fast fp128 - // AARCH64-FASTMATH: fdiv fast fp128 + // |C| + // AARCH64-FASTMATH: call {{.*}}fp128 @llvm.fabs.f128(fp128 {{.*}}) + // |D| + // AARCH64-FASTMATH-NEXT: call {{.*}}fp128 @llvm.fabs.f128(fp128 {{.*}}) + // AARCH64-FASTMATH-NEXT: fcmp {{.*}}ugt fp128 + // AARCH64-FASTMATH-NEXT: br i1 {{.*}}, label + // AARCH64-FASTMATH: abs_rhsr_greater_or_equal_abs_rhsi: + + // |C| >= |D| + // DdC=D/C + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // CpRD=C+CdC*D + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + + // A+BR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // B-AR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fsub {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: abs_rhsr_less_than_abs_rhsi: + + // |C| < |D| + // CdD=C/D + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // DpRC=D+CdD*C + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + + // (A*CdD+B)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // (BCdD-A)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fsub {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: complex_div: + // AARCH64-FASTMATH-NEXT: phi {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: phi {{.*}}fp128 // AARCH64-FASTMATH: ret return a / b; } @@ -537,23 +758,68 @@ long double _Complex div_long_double_cc(long double _Complex a, long double _Com // PPC: ret // SPIR: call spir_func {{.*}} @__divdc3( - // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD)) + // a / b = (A+iB) / (C+iD) = (E+iF) + // if (|C| >= |D|) + // DdC = D/C + // CpRD = C+DdC*D + // E = (A+B*DdC)/CpRD + // F = (B-A*DdC)/CpRD + // else + // CdD = C/D + // DpRC= D+CdD*C + // E = (A*CdD+B)/DpRC + // F = (B*CdD-A)/DpRC // AARCH64-FASTMATH-LABEL: @div_long_double_cc([2 x fp128] noundef nofpclass(nan inf) alignstack(16) %a.coerce, [2 x fp128] noundef nofpclass(nan inf) alignstack(16) %b.coerce) - // - // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[BD:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[ACpBD:%.*]] = fadd fast fp128 - // - // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast fp128 - // - // AARCH64-FASTMATH: [[BC:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[BCmAD:%.*]] = fsub fast fp128 - // - // AARCH64-FASTMATH: fdiv fast fp128 - // AARCH64-FASTMATH: fdiv fast fp128 + // |C| + // AARCH64-FASTMATH: call {{.*}}fp128 @llvm.fabs.f128(fp128 {{.*}}) + // |D| + // AARCH64-FASTMATH-NEXT: call {{.*}}fp128 @llvm.fabs.f128(fp128 {{.*}}) + // AARCH64-FASTMATH-NEXT: fcmp {{.*}}ugt fp128 + // AARCH64-FASTMATH-NEXT: br i1 {{.*}}, label + // AARCH64-FASTMATH: abs_rhsr_greater_or_equal_abs_rhsi: + + // |C| >= |D| + // DdC=D/C + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // CpRD=C+CdC*D + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + + // A+BR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // B-AR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fsub {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: abs_rhsr_less_than_abs_rhsi: + + // |C| < |D| + // CdD=C/D + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // DpRC=D+CdD*C + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + + // (A*CdD+B)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // (BCdD-A)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fsub {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: complex_div: + // AARCH64-FASTMATH-NEXT: phi {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: phi {{.*}}fp128 // AARCH64-FASTMATH: ret return a / b; } diff --git a/clang/test/CodeGen/cx-complex-range.c b/clang/test/CodeGen/cx-complex-range.c new file mode 100644 index 0000000000000000000000000000000000000000..8368fa611335cca7595e921fd754119b5119a3c4 --- /dev/null +++ b/clang/test/CodeGen/cx-complex-range.c @@ -0,0 +1,108 @@ +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -o - | FileCheck %s --check-prefix=FULL + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=limited -o - | FileCheck %s --check-prefix=LMTD + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -fno-cx-limited-range -o - | FileCheck %s --check-prefix=FULL + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=fortran -o - | FileCheck %s --check-prefix=FRTRN + +// Fast math +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu \ +// RUN: -ffast-math -complex-range=limited -emit-llvm -o - %s \ +// RUN: | FileCheck %s --check-prefix=LMTD-FAST + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -fno-cx-fortran-rules -o - | FileCheck %s --check-prefix=FULL + +_Complex float div(_Complex float a, _Complex float b) { + // LABEL: define {{.*}} @div( + // FULL: call {{.*}} @__divsc3 + + // LMTD: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fadd float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fadd float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fsub float + // LMTD-NEXT: fdiv float + // LMTD-NEXT: fdiv float + + // FRTRN: call {{.*}}float @llvm.fabs.f32(float {{.*}}) + // FRTRN-NEXT: call {{.*}}float @llvm.fabs.f32(float {{.*}}) + // FRTRN-NEXT: fcmp {{.*}}ugt float + // FRTRN-NEXT: br i1 {{.*}}, label + // FRTRN: abs_rhsr_greater_or_equal_abs_rhsi: + // FRTRN-NEXT: fdiv {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fadd {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fadd {{.*}}float + // FRTRN-NEXT: fdiv {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fsub {{.*}}float + // FRTRN-NEXT: fdiv {{.*}}float + // FRTRN-NEXT: br label + // FRTRN: abs_rhsr_less_than_abs_rhsi: + // FRTRN-NEXT: fdiv {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fadd {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fadd {{.*}}float + // FRTRN-NEXT: fdiv {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fsub {{.*}}float + // FRTRN-NEXT: fdiv {{.*}}float + // FRTRN-NEXT: br label + // FRTRN: complex_div: + // FRTRN-NEXT: phi {{.*}}float + // FRTRN-NEXT: phi {{.*}}float + + // LMTD-FAST: fmul {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fadd {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fadd {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fsub {{.*}} float + // LMTD-FAST-NEXT: fdiv {{.*}} float + // LMTD-FAST-NEXT: fdiv {{.*}} float + + return a / b; +} + +_Complex float mul(_Complex float a, _Complex float b) { + // LABEL: define {{.*}} @mul( + // FULL: call {{.*}} @__mulsc3 + + // LMTD: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fsub float + // LMTD-NEXT: fadd float + + // FRTRN: fmul {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fsub {{.*}}float + // FRTRN-NEXT: fadd {{.*}}float + + // LMTD-FAST: fmul {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fsub {{.*}} float + // LMTD-FAST-NEXT: fadd {{.*}} float + + return a * b; +} diff --git a/clang/test/CodeGen/pgo-instrumentation.c b/clang/test/CodeGen/pgo-instrumentation.c index a65c6712291bd26c414e88d29498339f599ada6e..c01658065497e3036105f388e399e57c86904650 100644 --- a/clang/test/CodeGen/pgo-instrumentation.c +++ b/clang/test/CodeGen/pgo-instrumentation.c @@ -3,7 +3,7 @@ // Ensure Pass PGOInstrumentationGenPass is invoked. // RUN: %clang_cc1 -O2 -fprofile-instrument=llvm %s -fdebug-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s -check-prefix=CHECK-PGOGENPASS-INVOKED-INSTR-GEN --check-prefix=CHECK-INSTRPROF // CHECK-PGOGENPASS-INVOKED-INSTR-GEN: Running pass: PGOInstrumentationGen on -// CHECK-INSTRPROF: Running pass: InstrProfiling on +// CHECK-INSTRPROF: Running pass: InstrProfilingLoweringPass on // // Ensure Pass PGOInstrumentationGenPass is not invoked. // RUN: %clang_cc1 -O2 -fprofile-instrument=clang %s -fdebug-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s -check-prefix=CHECK-PGOGENPASS-INVOKED-INSTR-GEN-CLANG @@ -11,7 +11,7 @@ // RUN: %clang_cc1 -O2 -fprofile-instrument=clang %s -fdebug-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-CLANG-INSTRPROF // RUN: %clang_cc1 -O0 -fprofile-instrument=clang %s -fdebug-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-CLANG-INSTRPROF -// CHECK-CLANG-INSTRPROF: Running pass: InstrProfiling on +// CHECK-CLANG-INSTRPROF: Running pass: InstrProfilingLoweringPass on // Ensure Pass PGOInstrumentationUsePass is invoked. // RUN: llvm-profdata merge -o %t.profdata %S/Inputs/pgotestir.profraw diff --git a/clang/test/CodeGen/pragma-cx-limited-range.c b/clang/test/CodeGen/pragma-cx-limited-range.c new file mode 100644 index 0000000000000000000000000000000000000000..926da8afbee558c9428b0ed2e41605597b6e6a58 --- /dev/null +++ b/clang/test/CodeGen/pragma-cx-limited-range.c @@ -0,0 +1,107 @@ +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -o - | FileCheck %s --check-prefix=FULL + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=limited -o - | FileCheck --check-prefix=LMTD %s + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -fno-cx-limited-range -o - | FileCheck %s --check-prefix=FULL + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=fortran -o - | FileCheck --check-prefix=FRTRN %s + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -fno-cx-fortran-rules -o - | FileCheck --check-prefix=FULL %s + +_Complex float pragma_on_mul(_Complex float a, _Complex float b) { +#pragma STDC CX_LIMITED_RANGE ON + // LABEL: define {{.*}} @pragma_on_mul( + // FULL: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fsub float + // FULL-NEXT: fadd float + + // LMTD: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fsub float + // LMTD-NEXT: fadd float + + // FRTRN: fmul float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fsub float + // FRTRN-NEXT: fadd float + + return a * b; +} + +_Complex float pragma_off_mul(_Complex float a, _Complex float b) { +#pragma STDC CX_LIMITED_RANGE OFF + // LABEL: define {{.*}} @pragma_off_mul( + // FULL: call {{.*}} @__mulsc3 + + // LMTD: call {{.*}} @__mulsc3 + + // FRTRN: call {{.*}} @__mulsc3 + + return a * b; +} + +_Complex float pragma_on_div(_Complex float a, _Complex float b) { +#pragma STDC CX_LIMITED_RANGE ON + // LABEL: define {{.*}} @pragma_on_div( + // FULL: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fadd float + // FULL-NEXT: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fadd float + // FULL-NEXT: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fsub float + // FULL-NEXT: fdiv float + // FULL: fdiv float + + // LMTD: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fadd float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fadd float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fsub float + // LMTD-NEXT: fdiv float + // LMTD-NEXT: fdiv float + + // FRTRN: fmul float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fadd float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fadd float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fsub float + // FRTRN-NEXT: fdiv float + // FRTRN-NEXT: fdiv float + + return a / b; +} + +_Complex float pragma_off_div(_Complex float a, _Complex float b) { +#pragma STDC CX_LIMITED_RANGE OFF + // LABEL: define {{.*}} @pragma_off_div( + // FULL: call {{.*}} @__divsc3 + + // LMTD: call {{.*}} @__divsc3 + + // FRTRN: call {{.*}} @__divsc3 + + return a / b; +} diff --git a/clang/test/CodeGen/scoped-atomic-ops.c b/clang/test/CodeGen/scoped-atomic-ops.c new file mode 100644 index 0000000000000000000000000000000000000000..b0032046639b89eb067bef4aca671e2f9f9a3767 --- /dev/null +++ b/clang/test/CodeGen/scoped-atomic-ops.c @@ -0,0 +1,331 @@ +// RUN: %clang_cc1 %s -emit-llvm -o - -triple=amdgcn-amd-amdhsa -ffreestanding \ +// RUN: -fvisibility=hidden | FileCheck %s + +// CHECK-LABEL: define hidden i32 @fi1a( +// CHECK: [[TMP0:%.*]] = load atomic i32, ptr [[PTR0:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = load atomic i32, ptr [[PTR1:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = load atomic i32, ptr [[PTR2:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = load atomic i32, ptr [[PTR3:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = load atomic i32, ptr [[PTR4:.+]] syncscope("singlethread-one-as") monotonic, align 4 +int fi1a(int *i) { + int v; + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + return v; +} + +// CHECK-LABEL: define hidden i32 @fi1b( +// CHECK: [[TMP0:%.*]] = load atomic i32, ptr [[PTR0:%.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = load atomic i32, ptr [[PTR1:%.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = load atomic i32, ptr [[PTR2:%.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = load atomic i32, ptr [[PTR3:%.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = load atomic i32, ptr [[PTR4:%.+]] syncscope("singlethread-one-as") monotonic, align 4 +// +int fi1b(int *i) { + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + return *i; +} + +// CHECK-LABEL: define hidden void @fi2a( +// CHECK: store atomic i32 [[TMP0:%.+]], ptr [[PTR0:%.+]] syncscope("one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP1:%.+]], ptr [[PTR1:%.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP2:%.+]], ptr [[PTR2:%.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP3:%.+]], ptr [[PTR3:%.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP4:%.+]], ptr [[PTR4:%.+]] syncscope("singlethread-one-as") monotonic, align 4 +// +void fi2a(int *i) { + int v = 1; + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden void @fi2b( +// CHECK: store atomic i32 [[TMP0:%.+]], ptr [[PTR0:%.+]] syncscope("one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP1:%.+]], ptr [[PTR1:%.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP2:%.+]], ptr [[PTR2:%.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP3:%.+]], ptr [[PTR3:%.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP4:%.+]], ptr [[PTR4:%.+]] syncscope("singlethread-one-as") monotonic, align 4 +void fi2b(int *i) { + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden void @fi3a( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("one-as") monotonic, align 4 +void fi3a(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); +} + +// CHECK-LABEL: define hidden void @fi3b( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("agent-one-as") monotonic, align 4 +void fi3b(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); +} + +// CHECK-LABEL: define hidden void @fi3c( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("workgroup-one-as") monotonic, align 4 +void fi3c(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); +} + +// CHECK-LABEL: define hidden void @fi3d( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("wavefront-one-as") monotonic, align 4 +void fi3d(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); +} + +// CHECK-LABEL: define hidden void @fi3e( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("singlethread-one-as") monotonic, align 4 +void fi3e(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4a( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("one-as") acquire acquire, align 4 +_Bool fi4a(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SYSTEM); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4b( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("agent-one-as") acquire acquire, align 4 +_Bool fi4b(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_DEVICE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4c( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("workgroup-one-as") acquire acquire, align 4 +_Bool fi4c(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_WRKGRP); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4d( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("wavefront-one-as") acquire acquire, align 4 +_Bool fi4d(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_WVFRNT); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4e( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("singlethread-one-as") acquire acquire, align 4 +_Bool fi4e(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5a( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("one-as") acquire acquire, align 4 +_Bool fi5a(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n(i, &cmp, 1, 1, __ATOMIC_ACQUIRE, + __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SYSTEM); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5b( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("agent-one-as") acquire acquire, align 4 +_Bool fi5b(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n(i, &cmp, 1, 1, __ATOMIC_ACQUIRE, + __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_DEVICE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5c( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("workgroup-one-as") acquire acquire, align 4 +_Bool fi5c(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n( + i, &cmp, 1, 1, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_WRKGRP); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5d( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("wavefront-one-as") acquire acquire, align 4 +_Bool fi5d(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n( + i, &cmp, 1, 1, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_WVFRNT); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5e( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("singlethread-one-as") acquire acquire, align 4 +_Bool fi5e(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n( + i, &cmp, 1, 1, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden i32 @fi6a( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("one-as") monotonic, align 4 +int fi6a(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return ret; +} + +// CHECK-LABEL: define hidden i32 @fi6b( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("agent-one-as") monotonic, align 4 +int fi6b(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + return ret; +} + +// CHECK-LABEL: define hidden i32 @fi6c( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("workgroup-one-as") monotonic, align 4 +int fi6c(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + return ret; +} + +// CHECK-LABEL: define hidden i32 @fi6d( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("wavefront-one-as") monotonic, align 4 +int fi6d(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + return ret; +} + +// CHECK-LABEL: define hidden i32 @fi6e( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("singlethread-one-as") monotonic, align 4 +int fi6e(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + return ret; +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7a( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("one-as") monotonic, align 1 +_Bool fi7a(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_SYSTEM); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7b( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("agent-one-as") monotonic, align 1 +_Bool fi7b(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_DEVICE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7c( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("workgroup-one-as") monotonic, align 1 +_Bool fi7c(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_WRKGRP); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7d( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("wavefront-one-as") monotonic, align 1 +_Bool fi7d(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_WVFRNT); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7e( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("singlethread-one-as") monotonic, align 1 +_Bool fi7e(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_SINGLE); +} diff --git a/clang/test/CodeGen/tbaa.c b/clang/test/CodeGen/tbaa.c new file mode 100644 index 0000000000000000000000000000000000000000..0ab81f60a71941c56809271ee99299aaa9124d66 --- /dev/null +++ b/clang/test/CodeGen/tbaa.c @@ -0,0 +1,116 @@ +// RUN: %clang_cc1 -triple x86_64-apple-darwin -O1 -no-struct-path-tbaa -disable-llvm-passes %s -emit-llvm -o - | FileCheck %s +// RUN: %clang_cc1 -triple x86_64-apple-darwin -O1 -disable-llvm-passes %s -emit-llvm -o - | FileCheck %s -check-prefixes=PATH +// RUN: %clang_cc1 -triple x86_64-apple-darwin -O0 -disable-llvm-passes %s -emit-llvm -o - | FileCheck %s -check-prefix=NO-TBAA +// RUN: %clang_cc1 -triple x86_64-apple-darwin -O1 -relaxed-aliasing -disable-llvm-passes %s -emit-llvm -o - | FileCheck %s -check-prefix=NO-TBAA +// Test TBAA metadata generated by front-end. +// +// NO-TBAA-NOT: !tbaa + +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; + +typedef enum { + RED_AUTO_32, + GREEN_AUTO_32, + BLUE_AUTO_32 +} EnumAuto32; + +typedef enum { + RED_AUTO_64, + GREEN_AUTO_64, + BLUE_AUTO_64 = 0x100000000ull +} EnumAuto64; + +typedef enum : uint16_t { + RED_16, + GREEN_16, + BLUE_16 +} Enum16; + +typedef enum : uint8_t { + RED_8, + GREEN_8, + BLUE_8 +} Enum8; + +uint32_t g0(EnumAuto32 *E, uint32_t *val) { +// CHECK-LABEL: define{{.*}} i32 @g0( +// CHECK: store i32 5, ptr %{{.*}}, align 4, !tbaa [[TAG_i32:!.*]] +// CHECK: store i32 0, ptr %{{.*}}, align 4, !tbaa [[TAG_i32]] +// CHECK: load i32, ptr %{{.*}}, align 4, !tbaa [[TAG_i32]] +// PATH-LABEL: define{{.*}} i32 @g0( +// PATH: store i32 5, ptr %{{.*}}, align 4, !tbaa [[TAG_i32:!.*]] +// PATH: store i32 0, ptr %{{.*}}, align 4, !tbaa [[TAG_i32]] +// PATH: load i32, ptr %{{.*}}, align 4, !tbaa [[TAG_i32]] + *val = 5; + *E = RED_AUTO_32; + return *val; +} + +uint64_t g1(EnumAuto64 *E, uint64_t *val) { +// CHECK-LABEL: define{{.*}} i64 @g1( +// CHECK: store i64 5, ptr %{{.*}}, align 8, !tbaa [[TAG_i64:!.*]] +// CHECK: store i64 0, ptr %{{.*}}, align 8, !tbaa [[TAG_long:!.*]] +// CHECK: load i64, ptr %{{.*}}, align 8, !tbaa [[TAG_i64]] +// PATH-LABEL: define{{.*}} i64 @g1( +// PATH: store i64 5, ptr %{{.*}}, align 8, !tbaa [[TAG_i64:!.*]] +// PATH: store i64 0, ptr %{{.*}}, align 8, !tbaa [[TAG_long:!.*]] +// PATH: load i64, ptr %{{.*}}, align 8, !tbaa [[TAG_i64]] + *val = 5; + *E = RED_AUTO_64; + return *val; +} + +uint16_t g2(Enum16 *E, uint16_t *val) { +// CHECK-LABEL: define{{.*}} i16 @g2( +// CHECK: store i16 5, ptr %{{.*}}, align 2, !tbaa [[TAG_i16:!.*]] +// CHECK: store i16 0, ptr %{{.*}}, align 2, !tbaa [[TAG_i16]] +// CHECK: load i16, ptr %{{.*}}, align 2, !tbaa [[TAG_i16]] +// PATH-LABEL: define{{.*}} i16 @g2( +// PATH: store i16 5, ptr %{{.*}}, align 2, !tbaa [[TAG_i16:!.*]] +// PATH: store i16 0, ptr %{{.*}}, align 2, !tbaa [[TAG_i16]] +// PATH: load i16, ptr %{{.*}}, align 2, !tbaa [[TAG_i16]] + *val = 5; + *E = RED_16; + return *val; +} + +uint8_t g3(Enum8 *E, uint8_t *val) { +// CHECK-LABEL: define{{.*}} i8 @g3( +// CHECK: store i8 5, ptr %{{.*}}, align 1, !tbaa [[TAG_i8:!.*]] +// CHECK: store i8 0, ptr %{{.*}}, align 1, !tbaa [[TAG_i8]] +// CHECK: load i8, ptr %{{.*}}, align 1, !tbaa [[TAG_i8]] +// PATH-LABEL: define{{.*}} i8 @g3( +// PATH: store i8 5, ptr %{{.*}}, align 1, !tbaa [[TAG_i8:!.*]] +// PATH: store i8 0, ptr %{{.*}}, align 1, !tbaa [[TAG_i8]] +// PATH: load i8, ptr %{{.*}}, align 1, !tbaa [[TAG_i8]] + *val = 5; + *E = RED_8; + return *val; +} + +// CHECK: [[TYPE_char:!.*]] = !{!"omnipotent char", [[TAG_c_tbaa:!.*]], +// CHECK: [[TAG_c_tbaa]] = !{!"Simple C/C++ TBAA"} +// CHECK: [[TAG_i32]] = !{[[TYPE_i32:!.*]], [[TYPE_i32]], i64 0} +// CHECK: [[TYPE_i32]] = !{!"int", [[TYPE_char]], +// CHECK: [[TAG_i64]] = !{[[TYPE_i64:!.*]], [[TYPE_i64]], i64 0} +// CHECK: [[TYPE_i64]] = !{!"long long", [[TYPE_char]], +// CHECK: [[TAG_long]] = !{[[TYPE_long:!.*]], [[TYPE_long]], i64 0} +// CHECK: [[TYPE_long]] = !{!"long", [[TYPE_char]], +// CHECK: [[TAG_i16]] = !{[[TYPE_i16:!.*]], [[TYPE_i16]], i64 0} +// CHECK: [[TYPE_i16]] = !{!"short", [[TYPE_char]], +// CHECK: [[TAG_i8]] = !{[[TYPE_i8:!.*]], [[TYPE_char]], i64 0} + +// PATH: [[TYPE_char:!.*]] = !{!"omnipotent char", [[TAG_c_tbaa:!.*]], +// PATH: [[TAG_c_tbaa]] = !{!"Simple C/C++ TBAA"} +// PATH: [[TAG_i32]] = !{[[TYPE_i32:!.*]], [[TYPE_i32]], i64 0} +// PATH: [[TYPE_i32]] = !{!"int", [[TYPE_char]], +// PATH: [[TAG_i64]] = !{[[TYPE_i64:!.*]], [[TYPE_i64]], i64 0} +// PATH: [[TYPE_i64]] = !{!"long long", [[TYPE_char]], +// PATH: [[TAG_long]] = !{[[TYPE_long:!.*]], [[TYPE_long]], i64 0} +// PATH: [[TYPE_long]] = !{!"long", [[TYPE_char]], +// PATH: [[TAG_i16]] = !{[[TYPE_i16:!.*]], [[TYPE_i16]], i64 0} +// PATH: [[TYPE_i16]] = !{!"short", [[TYPE_char]], +// PATH: [[TAG_i8]] = !{[[TYPE_i8:!.*]], [[TYPE_char]], i64 0} diff --git a/clang/test/CodeGenCUDA/offloading-entries.cu b/clang/test/CodeGenCUDA/offloading-entries.cu index 46235051f1e4f12cda5812150400c08d69bd23b0..4f5cf65ecd0bde66e36b46ed351ca29548668c12 100644 --- a/clang/test/CodeGenCUDA/offloading-entries.cu +++ b/clang/test/CodeGenCUDA/offloading-entries.cu @@ -17,31 +17,47 @@ //. // CUDA: @.omp_offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00" // CUDA: @.omp_offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z18__device_stub__foov, ptr @.omp_offloading.entry_name, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries", align 1 -// CUDA: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [8 x i8] c"_Z3barv\00" -// CUDA: @.omp_offloading.entry._Z3barv = weak constant %struct.__tgt_offload_entry { ptr @_Z18__device_stub__barv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries", align 1 -// CUDA: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [2 x i8] c"x\00" -// CUDA: @.omp_offloading.entry.x = weak constant %struct.__tgt_offload_entry { ptr @x, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "cuda_offloading_entries", align 1 +// CUDA: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00" +// CUDA: @.omp_offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z21__device_stub__kernelv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries", align 1 +// CUDA: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00" +// CUDA: @.omp_offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "cuda_offloading_entries", align 1 +// CUDA: @.omp_offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00" +// CUDA: @.omp_offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.omp_offloading.entry_name.3, i64 4, i32 2, i32 1 }, section "cuda_offloading_entries", align 1 +// CUDA: @.omp_offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00" +// CUDA: @.omp_offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.omp_offloading.entry_name.4, i64 4, i32 3, i32 1 }, section "cuda_offloading_entries", align 1 //. // HIP: @.omp_offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00" // HIP: @.omp_offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z3foov, ptr @.omp_offloading.entry_name, i64 0, i32 0, i32 0 }, section "hip_offloading_entries", align 1 -// HIP: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [8 x i8] c"_Z3barv\00" -// HIP: @.omp_offloading.entry._Z3barv = weak constant %struct.__tgt_offload_entry { ptr @_Z3barv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "hip_offloading_entries", align 1 -// HIP: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [2 x i8] c"x\00" -// HIP: @.omp_offloading.entry.x = weak constant %struct.__tgt_offload_entry { ptr @x, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "hip_offloading_entries", align 1 +// HIP: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00" +// HIP: @.omp_offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z6kernelv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "hip_offloading_entries", align 1 +// HIP: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00" +// HIP: @.omp_offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "hip_offloading_entries", align 1 +// HIP: @.omp_offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00" +// HIP: @.omp_offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.omp_offloading.entry_name.3, i64 4, i32 2, i32 1 }, section "hip_offloading_entries", align 1 +// HIP: @.omp_offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00" +// HIP: @.omp_offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.omp_offloading.entry_name.4, i64 4, i32 3, i32 1 }, section "hip_offloading_entries", align 1 //. // CUDA-COFF: @.omp_offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00" // CUDA-COFF: @.omp_offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z18__device_stub__foov, ptr @.omp_offloading.entry_name, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1 -// CUDA-COFF: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [8 x i8] c"_Z3barv\00" -// CUDA-COFF: @.omp_offloading.entry._Z3barv = weak constant %struct.__tgt_offload_entry { ptr @_Z18__device_stub__barv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1 -// CUDA-COFF: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [2 x i8] c"x\00" -// CUDA-COFF: @.omp_offloading.entry.x = weak constant %struct.__tgt_offload_entry { ptr @x, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1 +// CUDA-COFF: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00" +// CUDA-COFF: @.omp_offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z21__device_stub__kernelv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1 +// CUDA-COFF: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00" +// CUDA-COFF: @.omp_offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1 +// CUDA-COFF: @.omp_offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00" +// CUDA-COFF: @.omp_offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.omp_offloading.entry_name.3, i64 4, i32 2, i32 1 }, section "cuda_offloading_entries$OE", align 1 +// CUDA-COFF: @.omp_offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00" +// CUDA-COFF: @.omp_offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.omp_offloading.entry_name.4, i64 4, i32 3, i32 1 }, section "cuda_offloading_entries$OE", align 1 //. // HIP-COFF: @.omp_offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00" // HIP-COFF: @.omp_offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z3foov, ptr @.omp_offloading.entry_name, i64 0, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1 -// HIP-COFF: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [8 x i8] c"_Z3barv\00" -// HIP-COFF: @.omp_offloading.entry._Z3barv = weak constant %struct.__tgt_offload_entry { ptr @_Z3barv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1 -// HIP-COFF: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [2 x i8] c"x\00" -// HIP-COFF: @.omp_offloading.entry.x = weak constant %struct.__tgt_offload_entry { ptr @x, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1 +// HIP-COFF: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00" +// HIP-COFF: @.omp_offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z6kernelv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1 +// HIP-COFF: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00" +// HIP-COFF: @.omp_offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1 +// HIP-COFF: @.omp_offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00" +// HIP-COFF: @.omp_offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.omp_offloading.entry_name.3, i64 4, i32 2, i32 1 }, section "hip_offloading_entries$OE", align 1 +// HIP-COFF: @.omp_offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00" +// HIP-COFF: @.omp_offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.omp_offloading.entry_name.4, i64 4, i32 3, i32 1 }, section "hip_offloading_entries$OE", align 1 //. // CUDA-LABEL: @_Z18__device_stub__foov( // CUDA-NEXT: entry: @@ -72,34 +88,52 @@ // HIP-COFF-NEXT: ret void // __global__ void foo() {} +__device__ int var = 1; +const __device__ int constant = 1; +extern __device__ int external; -// CUDA-LABEL: @_Z18__device_stub__barv( +// CUDA-LABEL: @_Z21__device_stub__kernelv( // CUDA-NEXT: entry: -// CUDA-NEXT: [[TMP0:%.*]] = call i32 @cudaLaunch(ptr @_Z18__device_stub__barv) +// CUDA-NEXT: [[TMP0:%.*]] = call i32 @cudaLaunch(ptr @_Z21__device_stub__kernelv) // CUDA-NEXT: br label [[SETUP_END:%.*]] // CUDA: setup.end: // CUDA-NEXT: ret void // -// HIP-LABEL: @_Z18__device_stub__barv( +// HIP-LABEL: @_Z21__device_stub__kernelv( // HIP-NEXT: entry: -// HIP-NEXT: [[TMP0:%.*]] = call i32 @hipLaunchByPtr(ptr @_Z3barv) +// HIP-NEXT: [[TMP0:%.*]] = call i32 @hipLaunchByPtr(ptr @_Z6kernelv) // HIP-NEXT: br label [[SETUP_END:%.*]] // HIP: setup.end: // HIP-NEXT: ret void // -// CUDA-COFF-LABEL: @_Z18__device_stub__barv( +// CUDA-COFF-LABEL: @_Z21__device_stub__kernelv( // CUDA-COFF-NEXT: entry: -// CUDA-COFF-NEXT: [[TMP0:%.*]] = call i32 @cudaLaunch(ptr @_Z18__device_stub__barv) +// CUDA-COFF-NEXT: [[TMP0:%.*]] = call i32 @cudaLaunch(ptr @_Z21__device_stub__kernelv) // CUDA-COFF-NEXT: br label [[SETUP_END:%.*]] // CUDA-COFF: setup.end: // CUDA-COFF-NEXT: ret void // -// HIP-COFF-LABEL: @_Z18__device_stub__barv( +// HIP-COFF-LABEL: @_Z21__device_stub__kernelv( // HIP-COFF-NEXT: entry: -// HIP-COFF-NEXT: [[TMP0:%.*]] = call i32 @hipLaunchByPtr(ptr @_Z3barv) +// HIP-COFF-NEXT: [[TMP0:%.*]] = call i32 @hipLaunchByPtr(ptr @_Z6kernelv) // HIP-COFF-NEXT: br label [[SETUP_END:%.*]] // HIP-COFF: setup.end: // HIP-COFF-NEXT: ret void // -__global__ void bar() {} -__device__ int x = 1; +__global__ void kernel() { external = 1; } + +struct surfaceReference { int desc; }; + +template +struct __attribute__((device_builtin_surface_type)) surface : public surfaceReference {}; + +surface surf; + +struct textureReference { + int desc; +}; + +template +struct __attribute__((device_builtin_texture_type)) texture : public textureReference {}; + +texture tex; diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl index 77091f8390a15a18e39253af82a8c42f7bc9e8c8..a70e224b81e4b74ab36f6e97cc5a7058b12b5cee 100644 --- a/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl +++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl @@ -16,9 +16,9 @@ void main() { } // CHECK: !hlsl.uavs = !{![[Single:[0-9]+]], ![[Array:[0-9]+]], ![[SingleAllocated:[0-9]+]], ![[ArrayAllocated:[0-9]+]], ![[SingleSpace:[0-9]+]], ![[ArraySpace:[0-9]+]]} -// CHECK-DAG: ![[Single]] = !{ptr @"?Buffer1@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i32 -1, i32 0} -// CHECK-DAG: ![[Array]] = !{ptr @"?BufferArray@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i32 -1, i32 0} -// CHECK-DAG: ![[SingleAllocated]] = !{ptr @"?Buffer2@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i32 3, i32 0} -// CHECK-DAG: ![[ArrayAllocated]] = !{ptr @"?BufferArray2@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i32 4, i32 0} -// CHECK-DAG: ![[SingleSpace]] = !{ptr @"?Buffer3@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i32 3, i32 1} -// CHECK-DAG: ![[ArraySpace]] = !{ptr @"?BufferArray3@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i32 4, i32 1} +// CHECK-DAG: ![[Single]] = !{ptr @"?Buffer1@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i1 false, i32 -1, i32 0} +// CHECK-DAG: ![[Array]] = !{ptr @"?BufferArray@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i1 false, i32 -1, i32 0} +// CHECK-DAG: ![[SingleAllocated]] = !{ptr @"?Buffer2@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i1 false, i32 3, i32 0} +// CHECK-DAG: ![[ArrayAllocated]] = !{ptr @"?BufferArray2@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i1 false, i32 4, i32 0} +// CHECK-DAG: ![[SingleSpace]] = !{ptr @"?Buffer3@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i1 false, i32 3, i32 1} +// CHECK-DAG: ![[ArraySpace]] = !{ptr @"?BufferArray3@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i1 false, i32 4, i32 1} diff --git a/clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl new file mode 100644 index 0000000000000000000000000000000000000000..ce7d84ecf5b1470d1a6605bd9095b6f1e5130447 --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-pixel -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s + +RasterizerOrderedBuffer Buffer1; +RasterizerOrderedBuffer > BufferArray[4]; + +RasterizerOrderedBuffer Buffer2 : register(u3); +RasterizerOrderedBuffer > BufferArray2[4] : register(u4); + +RasterizerOrderedBuffer Buffer3 : register(u3, space1); +RasterizerOrderedBuffer > BufferArray3[4] : register(u4, space1); + +void main() {} + +// CHECK: !hlsl.uavs = !{![[Single:[0-9]+]], ![[Array:[0-9]+]], ![[SingleAllocated:[0-9]+]], ![[ArrayAllocated:[0-9]+]], ![[SingleSpace:[0-9]+]], ![[ArraySpace:[0-9]+]]} +// CHECK-DAG: ![[Single]] = !{ptr @"?Buffer1@@3V?$RasterizerOrderedBuffer@M@hlsl@@A", !"RasterizerOrderedBuffer", i32 10, i1 true, i32 -1, i32 0} +// CHECK-DAG: ![[Array]] = !{ptr @"?BufferArray@@3PAV?$RasterizerOrderedBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RasterizerOrderedBuffer >", i32 10, i1 true, i32 -1, i32 0} +// CHECK-DAG: ![[SingleAllocated]] = !{ptr @"?Buffer2@@3V?$RasterizerOrderedBuffer@M@hlsl@@A", !"RasterizerOrderedBuffer", i32 10, i1 true, i32 3, i32 0} +// CHECK-DAG: ![[ArrayAllocated]] = !{ptr @"?BufferArray2@@3PAV?$RasterizerOrderedBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RasterizerOrderedBuffer >", i32 10, i1 true, i32 4, i32 0} +// CHECK-DAG: ![[SingleSpace]] = !{ptr @"?Buffer3@@3V?$RasterizerOrderedBuffer@M@hlsl@@A", !"RasterizerOrderedBuffer", i32 10, i1 true, i32 3, i32 1} +// CHECK-DAG: ![[ArraySpace]] = !{ptr @"?BufferArray3@@3PAV?$RasterizerOrderedBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RasterizerOrderedBuffer >", i32 10, i1 true, i32 4, i32 1} diff --git a/clang/test/CodeGenHLSL/cbuf.hlsl b/clang/test/CodeGenHLSL/cbuf.hlsl index 92c883943d03e774829d137fff019741a74bad72..5dee1feb902aa0391951d611a8afc75f7f7e7a08 100644 --- a/clang/test/CodeGenHLSL/cbuf.hlsl +++ b/clang/test/CodeGenHLSL/cbuf.hlsl @@ -24,5 +24,5 @@ float foo() { // CHECK: !hlsl.cbufs = !{![[CBMD:[0-9]+]]} // CHECK: !hlsl.srvs = !{![[TBMD:[0-9]+]]} -// CHECK: ![[CBMD]] = !{ptr @[[CB]], !"A.cb.ty", i32 13, i32 0, i32 2} -// CHECK: ![[TBMD]] = !{ptr @[[TB]], !"A.tb.ty", i32 15, i32 2, i32 1} +// CHECK: ![[CBMD]] = !{ptr @[[CB]], !"A.cb.ty", i32 13, i1 false, i32 0, i32 2} +// CHECK: ![[TBMD]] = !{ptr @[[TB]], !"A.tb.ty", i32 15, i1 false, i32 2, i32 1} diff --git a/clang/test/Driver/aarch64-outliner.c b/clang/test/Driver/aarch64-outliner.c index 42e43b433e282d387457e633cd66829cf27c95d8..06e5de11ec49ecde2007cf53da388d116989be21 100644 --- a/clang/test/Driver/aarch64-outliner.c +++ b/clang/test/Driver/aarch64-outliner.c @@ -1,7 +1,9 @@ // REQUIRES: aarch64-registered-target // RUN: %clang --target=aarch64 -moutline -S %s -### 2>&1 | FileCheck %s -check-prefix=ON +// RUN: %clang --target=aarch64_be -moutline -S %s -### 2>&1 | FileCheck %s -check-prefix=ON // ON: "-mllvm" "-enable-machine-outliner" // RUN: %clang --target=aarch64 -moutline -mno-outline -S %s -### 2>&1 | FileCheck %s -check-prefix=OFF +// RUN: %clang --target=aarch64_be -moutline -mno-outline -S %s -### 2>&1 | FileCheck %s -check-prefix=OFF // OFF: "-mllvm" "-enable-machine-outliner=never" // RUN: %clang --target=x86_64 -moutline -S %s -### 2>&1 | FileCheck %s -check-prefix=WARN // WARN: warning: 'x86_64' does not support '-moutline'; flag ignored [-Woption-ignored] diff --git a/clang/test/Driver/arm-cortex-cpus-2.c b/clang/test/Driver/arm-cortex-cpus-2.c index 5b52488bc498f3e41b367719ebdcbee466715047..c322303d22786681c25d11b271f273624d5cc2d7 100644 --- a/clang/test/Driver/arm-cortex-cpus-2.c +++ b/clang/test/Driver/arm-cortex-cpus-2.c @@ -562,8 +562,11 @@ // RUN: %clang -target arm -mcpu=cortex-m85 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-M85 %s // CHECK-CORTEX-M85: "-cc1"{{.*}} "-triple" "thumbv8.1m.main-{{.*}} "-target-cpu" "cortex-m85" +// RUN: %clang -target arm -mcpu=cortex-m52 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-M52 %s +// CHECK-CORTEX-M52: "-cc1"{{.*}} "-triple" "thumbv8.1m.main-{{.*}} "-target-cpu" "cortex-m52" + // RUN: %clang -target arm -mcpu=neoverse-n2 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-NEOVERSE-N2 %s -// CHECK-NEOVERSE-N2: "-cc1"{{.*}} "-triple" "armv8.5a-{{.*}}" "-target-cpu" "neoverse-n2" +// CHECK-NEOVERSE-N2: "-cc1"{{.*}} "-triple" "armv9a-{{.*}}" "-target-cpu" "neoverse-n2" // ================== Check whether -mcpu accepts mixed-case values. // RUN: %clang -target arm-linux-gnueabi -mcpu=Cortex-a5 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CASE-INSENSITIVE-CPUV7A %s diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c index 6d929b19e7e2ef8da5fdc6b038bcf8b4a70433b1..81d1b907eced188c5c7096aca6e9c751f9973eb3 100644 --- a/clang/test/Driver/cl-options.c +++ b/clang/test/Driver/cl-options.c @@ -747,7 +747,7 @@ // Validate that the default triple is used when run an empty tools dir is specified // RUN: %clang_cl -vctoolsdir "" -### -- %s 2>&1 | FileCheck %s --check-prefix VCTOOLSDIR -// VCTOOLSDIR: "-triple" "{{[a-zA-Z0-9_-]*}}-pc-windows-msvc19.20.0" +// VCTOOLSDIR: "-triple" "{{[a-zA-Z0-9_-]*}}-pc-windows-msvc19.33.0" // Validate that built-in include paths are based on the supplied path // RUN: %clang_cl --target=aarch64-pc-windows-msvc -vctoolsdir "/fake" -winsdkdir "/foo" -winsdkversion 10.0.12345.0 -### -- %s 2>&1 | FileCheck %s --check-prefix FAKEDIR @@ -787,7 +787,7 @@ // RUN: %clang_cl -vctoolsdir "" /arm64EC /c -### -- %s 2>&1 | FileCheck --check-prefix=ARM64EC %s // ARM64EC-NOT: /arm64EC has been overridden by specified target -// ARM64EC: "-triple" "arm64ec-pc-windows-msvc19.20.0" +// ARM64EC: "-triple" "arm64ec-pc-windows-msvc19.33.0" // RUN: %clang_cl -vctoolsdir "" /arm64EC /c -target x86_64-pc-windows-msvc -### -- %s 2>&1 | FileCheck --check-prefix=ARM64EC_OVERRIDE %s // ARM64EC_OVERRIDE: warning: /arm64EC has been overridden by specified target: x86_64-pc-windows-msvc; option ignored diff --git a/clang/test/Driver/clang-offload-bundler-zlib.c b/clang/test/Driver/clang-offload-bundler-zlib.c index c46c32a4a0537b4047b77d0e61db65cadcfac162..a57ee6da9a86a6d31ecb1091335a0acbb5ce8929 100644 --- a/clang/test/Driver/clang-offload-bundler-zlib.c +++ b/clang/test/Driver/clang-offload-bundler-zlib.c @@ -55,14 +55,15 @@ // Check unbundling archive. // // RUN: clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \ -// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%T/hip_bundle1.bc -compress +// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%t.hip_bundle1.bc -compress // RUN: clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \ -// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%T/hip_bundle2.bc -compress -// RUN: llvm-ar cr %T/hip_archive.a %T/hip_bundle1.bc %T/hip_bundle2.bc +// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%t.hip_bundle2.bc -compress +// RUN: rm -f %t.hip_archive.a +// RUN: llvm-ar cr %t.hip_archive.a %t.hip_bundle1.bc %t.hip_bundle2.bc // RUN: clang-offload-bundler -unbundle -type=a -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \ -// RUN: -output=%T/hip_900.a -output=%T/hip_906.a -input=%T/hip_archive.a -// RUN: llvm-ar t %T/hip_900.a | FileCheck -check-prefix=HIP-AR-900 %s -// RUN: llvm-ar t %T/hip_906.a | FileCheck -check-prefix=HIP-AR-906 %s +// RUN: -output=%t.hip_900.a -output=%t.hip_906.a -input=%t.hip_archive.a +// RUN: llvm-ar t %t.hip_900.a | FileCheck -check-prefix=HIP-AR-900 %s +// RUN: llvm-ar t %t.hip_906.a | FileCheck -check-prefix=HIP-AR-906 %s // HIP-AR-900-DAG: hip_bundle1-hip-amdgcn-amd-amdhsa--gfx900 // HIP-AR-900-DAG: hip_bundle2-hip-amdgcn-amd-amdhsa--gfx900 // HIP-AR-906-DAG: hip_bundle1-hip-amdgcn-amd-amdhsa--gfx906 diff --git a/clang/test/Driver/clang-offload-bundler-zstd.c b/clang/test/Driver/clang-offload-bundler-zstd.c index b2b588b72d4d6f282b376b5c734cba9ea1eeb614..3b577d4d166a3f5e8b6162d67bcb3f981ad4f9ae 100644 --- a/clang/test/Driver/clang-offload-bundler-zstd.c +++ b/clang/test/Driver/clang-offload-bundler-zstd.c @@ -48,18 +48,18 @@ // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 -// // Check unbundling archive. // // RUN: clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \ -// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%T/hip_bundle1.bc -compress +// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%t.hip_bundle1.bc -compress // RUN: clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \ -// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%T/hip_bundle2.bc -compress -// RUN: llvm-ar cr %T/hip_archive.a %T/hip_bundle1.bc %T/hip_bundle2.bc +// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%t.hip_bundle2.bc -compress +// RUN: rm -f %t.hip_archive.a +// RUN: llvm-ar cr %t.hip_archive.a %t.hip_bundle1.bc %t.hip_bundle2.bc // RUN: clang-offload-bundler -unbundle -type=a -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \ -// RUN: -output=%T/hip_900.a -output=%T/hip_906.a -input=%T/hip_archive.a -// RUN: llvm-ar t %T/hip_900.a | FileCheck -check-prefix=HIP-AR-900 %s -// RUN: llvm-ar t %T/hip_906.a | FileCheck -check-prefix=HIP-AR-906 %s +// RUN: -output=%t.hip_900.a -output=%t.hip_906.a -input=%t.hip_archive.a +// RUN: llvm-ar t %t.hip_900.a | FileCheck -check-prefix=HIP-AR-900 %s +// RUN: llvm-ar t %t.hip_906.a | FileCheck -check-prefix=HIP-AR-906 %s // HIP-AR-900-DAG: hip_bundle1-hip-amdgcn-amd-amdhsa--gfx900 // HIP-AR-900-DAG: hip_bundle2-hip-amdgcn-amd-amdhsa--gfx900 // HIP-AR-906-DAG: hip_bundle1-hip-amdgcn-amd-amdhsa--gfx906 diff --git a/clang/test/Driver/fdefine-target-os-macros.c b/clang/test/Driver/fdefine-target-os-macros.c new file mode 100644 index 0000000000000000000000000000000000000000..030d4ce34cb282bd204eca00f21c9d20f854b9ea --- /dev/null +++ b/clang/test/Driver/fdefine-target-os-macros.c @@ -0,0 +1,250 @@ +// RUN: %clang -### --target=arm64-apple-darwin %s 2>&1 | FileCheck %s --check-prefix=DARWIN-DEFAULT +// DARWIN-DEFAULT-NOT: "-fdefine-target-os-macros" + +// RUN: %clang -### --target=arm-none-linux-gnu %s 2>&1 | FileCheck %s --check-prefix=NON-DARWIN-DEFAULT +// RUN: %clang -### --target=x86_64-pc-win32 %s 2>&1 | FileCheck %s --check-prefix=NON-DARWIN-DEFAULT +// NON-DARWIN-DEFAULT-NOT: "-fdefine-target-os-macros" + +// RUN: %clang -dM -E --target=arm64-apple-macos \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=1 \ +// RUN: -DIPHONE=0 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-ios \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=1 \ +// RUN: -DIOS=1 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=1 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-ios-macabi \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=1 \ +// RUN: -DIOS=1 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=1 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-ios-simulator \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=1 \ +// RUN: -DIOS=1 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=1 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-tvos \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=1 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=1 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=1 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-tvos-simulator \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=1 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=1 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=1 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-watchos \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=1 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=1 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=1 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-watchos-simulator \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=1 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=1 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=1 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-driverkit \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=0 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=1 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=x86_64-pc-linux-gnu \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=0 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=0 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=1 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=x86_64-pc-win32 \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=0 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=0 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=1 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=x86_64-pc-windows-gnu \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=0 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=0 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=1 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=sparc-none-solaris \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=0 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=0 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=1 + +// RUN: %clang -dM -E --target=arm64-apple-macos \ +// RUN: -fno-define-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s --check-prefix=NEG + +// RUN: %clang -dM -E --target=arm64-apple-macos \ +// RUN: -fdefine-target-os-macros \ +// RUN: -fno-define-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s --check-prefix=NEG + +// RUN: %clang -dM -E --target=x86_64-pc-windows \ +// RUN: -fdefine-target-os-macros \ +// RUN: -fno-define-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s --check-prefix=NEG + +// NEG-NOT: #define TARGET_OS_ + +// CHECK-DAG: #define TARGET_OS_MAC [[MAC]] +// CHECK-DAG: #define TARGET_OS_OSX [[OSX]] +// CHECK-DAG: #define TARGET_OS_IPHONE [[IPHONE]] +// CHECK-DAG: #define TARGET_OS_IOS [[IOS]] +// CHECK-DAG: #define TARGET_OS_TV [[TV]] +// CHECK-DAG: #define TARGET_OS_WATCH [[WATCH]] +// CHECK-DAG: #define TARGET_OS_DRIVERKIT [[DRIVERKIT]] +// CHECK-DAG: #define TARGET_OS_MACCATALYST [[MACCATALYST]] +// CHECK-DAG: #define TARGET_OS_SIMULATOR [[SIMULATOR]] +// Deprecated +// CHECK-DAG: #define TARGET_OS_EMBEDDED [[EMBEDDED]] +// CHECK-DAG: #define TARGET_OS_NANO [[WATCH]] +// CHECK-DAG: #define TARGET_IPHONE_SIMULATOR [[SIMULATOR]] +// CHECK-DAG: #define TARGET_OS_UIKITFORMAC [[MACCATALYST]] +// Non-darwin OSes +// CHECK-DAG: #define TARGET_OS_WIN32 [[WINDOWS]] +// CHECK-DAG: #define TARGET_OS_WINDOWS [[WINDOWS]] +// CHECK-DAG: #define TARGET_OS_LINUX [[LINUX]] +// CHECK-DAG: #define TARGET_OS_UNIX [[UNIX]] diff --git a/clang/test/Driver/hip-offload-compress-zlib.hip b/clang/test/Driver/hip-offload-compress-zlib.hip index a29b6d037350d47774a892f370b1a7f04d3396b9..7557fdde8786c7bfda882718e98ee7e0b9db3718 100644 --- a/clang/test/Driver/hip-offload-compress-zlib.hip +++ b/clang/test/Driver/hip-offload-compress-zlib.hip @@ -4,13 +4,13 @@ // Test compress bundled bitcode. -// RUN: rm -rf %T/a.bc +// RUN: rm -rf %t.bc // RUN: %clang -c -v --target=x86_64-linux-gnu \ // RUN: -x hip --offload-arch=gfx1100 --offload-arch=gfx1101 \ // RUN: -fgpu-rdc -nogpuinc -nogpulib \ // RUN: %S/Inputs/hip_multiple_inputs/a.cu \ // RUN: --offload-compress --offload-device-only --gpu-bundle-output \ -// RUN: -o %T/a.bc \ +// RUN: -o %t.bc \ // RUN: 2>&1 | FileCheck %s // CHECK: clang-offload-bundler{{.*}} -type=bc @@ -23,7 +23,7 @@ // RUN: %clang --hip-link -### -v --target=x86_64-linux-gnu \ // RUN: --offload-arch=gfx1100 --offload-arch=gfx1101 \ // RUN: -fgpu-rdc -nogpulib \ -// RUN: %T/a.bc --offload-device-only \ +// RUN: %t.bc --offload-device-only \ // RUN: 2>&1 | FileCheck -check-prefix=UNBUNDLE %s // UNBUNDLE: clang-offload-bundler{{.*}} "-type=bc" diff --git a/clang/test/Driver/hip-offload-compress-zstd.hip b/clang/test/Driver/hip-offload-compress-zstd.hip index 688c2c85329c1921ed120d97bf779f5c8ada984c..3680ae47974a601f4fc2c017d46a331828a19e63 100644 --- a/clang/test/Driver/hip-offload-compress-zstd.hip +++ b/clang/test/Driver/hip-offload-compress-zstd.hip @@ -4,13 +4,13 @@ // Test compress bundled bitcode. -// RUN: rm -rf %T/a.bc +// RUN: rm -rf %t.bc // RUN: %clang -c -v --target=x86_64-linux-gnu \ // RUN: -x hip --offload-arch=gfx1100 --offload-arch=gfx1101 \ // RUN: -fgpu-rdc -nogpuinc -nogpulib \ // RUN: %S/Inputs/hip_multiple_inputs/a.cu \ // RUN: --offload-compress --offload-device-only --gpu-bundle-output \ -// RUN: -o %T/a.bc \ +// RUN: -o %t.bc \ // RUN: 2>&1 | FileCheck %s // CHECK: clang-offload-bundler{{.*}} -type=bc @@ -23,7 +23,7 @@ // RUN: %clang --hip-link -### -v --target=x86_64-linux-gnu \ // RUN: --offload-arch=gfx1100 --offload-arch=gfx1101 \ // RUN: -fgpu-rdc -nogpulib \ -// RUN: %T/a.bc --offload-device-only \ +// RUN: %t.bc --offload-device-only \ // RUN: 2>&1 | FileCheck -check-prefix=UNBUNDLE %s // UNBUNDLE: clang-offload-bundler{{.*}} "-type=bc" diff --git a/clang/test/Driver/linker-wrapper-image.c b/clang/test/Driver/linker-wrapper-image.c index 73d3c40810c35a8ec0bf33842a1d76e7906f9e49..a2a1996f66430900e8ea655e978b6fe49052fa5f 100644 --- a/clang/test/Driver/linker-wrapper-image.c +++ b/clang/test/Driver/linker-wrapper-image.c @@ -80,24 +80,33 @@ // CUDA-NEXT: br i1 icmp ne (ptr @__start_cuda_offloading_entries, ptr @__stop_cuda_offloading_entries), label %while.entry, label %while.end // CUDA: while.entry: -// CUDA-NEXT: %entry1 = phi ptr [ @__start_cuda_offloading_entries, %entry ], [ %7, %if.end ] -// CUDA-NEXT: %1 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 0 -// CUDA-NEXT: %addr = load ptr, ptr %1, align 8 -// CUDA-NEXT: %2 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 1 -// CUDA-NEXT: %name = load ptr, ptr %2, align 8 -// CUDA-NEXT: %3 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 2 -// CUDA-NEXT: %size = load i64, ptr %3, align 4 -// CUDA-NEXT: %4 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 3 -// CUDA-NEXT: %flag = load i32, ptr %4, align 4 -// CUDA-NEXT: %5 = icmp eq i64 %size, 0 -// CUDA-NEXT: br i1 %5, label %if.then, label %if.else +// CUDA-NEXT: %entry1 = phi ptr [ @__start_cuda_offloading_entries, %entry ], [ %11, %if.end ] +// CUDA-NEXT: %1 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 0 +// CUDA-NEXT: %addr = load ptr, ptr %1, align 8 +// CUDA-NEXT: %2 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 1 +// CUDA-NEXT: %name = load ptr, ptr %2, align 8 +// CUDA-NEXT: %3 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 2 +// CUDA-NEXT: %size = load i64, ptr %3, align 4 +// CUDA-NEXT: %4 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 3 +// CUDA-NEXT: %flags = load i32, ptr %4, align 4 +// CUDA-NEXT: %5 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 4 +// CUDA-NEXT: %textype = load i32, ptr %5, align 4 +// CUDA-NEXT: %type = and i32 %flags, 7 +// CUDA-NEXT: %6 = and i32 %flags, 8 +// CUDA-NEXT: %extern = lshr i32 %6, 3 +// CUDA-NEXT: %7 = and i32 %flags, 16 +// CUDA-NEXT: %constant = lshr i32 %7, 4 +// CUDA-NEXT: %8 = and i32 %flags, 32 +// CUDA-NEXT: %normalized = lshr i32 %8, 5 +// CUDA-NEXT: %9 = icmp eq i64 %size, 0 +// CUDA-NEXT: br i1 %9, label %if.then, label %if.else // CUDA: if.then: -// CUDA-NEXT: %6 = call i32 @__cudaRegisterFunction(ptr %0, ptr %addr, ptr %name, ptr %name, i32 -1, ptr null, ptr null, ptr null, ptr null, ptr null) +// CUDA-NEXT: %10 = call i32 @__cudaRegisterFunction(ptr %0, ptr %addr, ptr %name, ptr %name, i32 -1, ptr null, ptr null, ptr null, ptr null, ptr null) // CUDA-NEXT: br label %if.end // CUDA: if.else: -// CUDA-NEXT: switch i32 %flag, label %if.end [ +// CUDA-NEXT: switch i32 %type, label %if.end [ // CUDA-NEXT: i32 0, label %sw.global // CUDA-NEXT: i32 1, label %sw.managed // CUDA-NEXT: i32 2, label %sw.surface @@ -105,22 +114,24 @@ // CUDA-NEXT: ] // CUDA: sw.global: -// CUDA-NEXT: call void @__cudaRegisterVar(ptr %0, ptr %addr, ptr %name, ptr %name, i32 0, i64 %size, i32 0, i32 0) +// CUDA-NEXT: call void @__cudaRegisterVar(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %extern, i64 %size, i32 %constant, i32 0) // CUDA-NEXT: br label %if.end // CUDA: sw.managed: // CUDA-NEXT: br label %if.end // CUDA: sw.surface: +// CUDA-NEXT: call void @__cudaRegisterSurface(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %textype, i32 %extern) // CUDA-NEXT: br label %if.end // CUDA: sw.texture: +// CUDA-NEXT: call void @__cudaRegisterTexture(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %textype, i32 %normalized, i32 %extern) // CUDA-NEXT: br label %if.end // CUDA: if.end: -// CUDA-NEXT: %7 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 1 -// CUDA-NEXT: %8 = icmp eq ptr %7, @__stop_cuda_offloading_entries -// CUDA-NEXT: br i1 %8, label %while.end, label %while.entry +// CUDA-NEXT: %11 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 1 +// CUDA-NEXT: %12 = icmp eq ptr %11, @__stop_cuda_offloading_entries +// CUDA-NEXT: br i1 %12, label %while.end, label %while.entry // CUDA: while.end: // CUDA-NEXT: ret void @@ -168,7 +179,7 @@ // HIP-NEXT: br i1 icmp ne (ptr @__start_hip_offloading_entries, ptr @__stop_hip_offloading_entries), label %while.entry, label %while.end // HIP: while.entry: -// HIP-NEXT: %entry1 = phi ptr [ @__start_hip_offloading_entries, %entry ], [ %7, %if.end ] +// HIP-NEXT: %entry1 = phi ptr [ @__start_hip_offloading_entries, %entry ], [ %11, %if.end ] // HIP-NEXT: %1 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 0 // HIP-NEXT: %addr = load ptr, ptr %1, align 8 // HIP-NEXT: %2 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 1 @@ -176,16 +187,25 @@ // HIP-NEXT: %3 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 2 // HIP-NEXT: %size = load i64, ptr %3, align 4 // HIP-NEXT: %4 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 3 -// HIP-NEXT: %flag = load i32, ptr %4, align 4 -// HIP-NEXT: %5 = icmp eq i64 %size, 0 -// HIP-NEXT: br i1 %5, label %if.then, label %if.else +// HIP-NEXT: %flags = load i32, ptr %4, align 4 +// HIP-NEXT: %5 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 4 +// HIP-NEXT: %textype = load i32, ptr %5, align 4 +// HIP-NEXT: %type = and i32 %flags, 7 +// HIP-NEXT: %6 = and i32 %flags, 8 +// HIP-NEXT: %extern = lshr i32 %6, 3 +// HIP-NEXT: %7 = and i32 %flags, 16 +// HIP-NEXT: %constant = lshr i32 %7, 4 +// HIP-NEXT: %8 = and i32 %flags, 32 +// HIP-NEXT: %normalized = lshr i32 %8, 5 +// HIP-NEXT: %9 = icmp eq i64 %size, 0 +// HIP-NEXT: br i1 %9, label %if.then, label %if.else // HIP: if.then: -// HIP-NEXT: %6 = call i32 @__hipRegisterFunction(ptr %0, ptr %addr, ptr %name, ptr %name, i32 -1, ptr null, ptr null, ptr null, ptr null, ptr null) +// HIP-NEXT: %10 = call i32 @__hipRegisterFunction(ptr %0, ptr %addr, ptr %name, ptr %name, i32 -1, ptr null, ptr null, ptr null, ptr null, ptr null) // HIP-NEXT: br label %if.end // HIP: if.else: -// HIP-NEXT: switch i32 %flag, label %if.end [ +// HIP-NEXT: switch i32 %type, label %if.end [ // HIP-NEXT: i32 0, label %sw.global // HIP-NEXT: i32 1, label %sw.managed // HIP-NEXT: i32 2, label %sw.surface @@ -193,22 +213,24 @@ // HIP-NEXT: ] // HIP: sw.global: -// HIP-NEXT: call void @__hipRegisterVar(ptr %0, ptr %addr, ptr %name, ptr %name, i32 0, i64 %size, i32 0, i32 0) +// HIP-NEXT: call void @__hipRegisterVar(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %extern, i64 %size, i32 %constant, i32 0) // HIP-NEXT: br label %if.end // HIP: sw.managed: // HIP-NEXT: br label %if.end // HIP: sw.surface: +// HIP-NEXT: call void @__hipRegisterSurface(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %textype, i32 %extern) // HIP-NEXT: br label %if.end // HIP: sw.texture: +// HIP-NEXT: call void @__hipRegisterTexture(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %textype, i32 %normalized, i32 %extern) // HIP-NEXT: br label %if.end // HIP: if.end: -// HIP-NEXT: %7 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 1 -// HIP-NEXT: %8 = icmp eq ptr %7, @__stop_hip_offloading_entries -// HIP-NEXT: br i1 %8, label %while.end, label %while.entry +// HIP-NEXT: %11 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 1 +// HIP-NEXT: %12 = icmp eq ptr %11, @__stop_hip_offloading_entries +// HIP-NEXT: br i1 %12, label %while.end, label %while.entry // HIP: while.end: // HIP-NEXT: ret void diff --git a/clang/test/Driver/linker-wrapper.c b/clang/test/Driver/linker-wrapper.c index e82febd618231024ccb41384a93f5f4c1ba3a03d..b763a003452ba721cff3bb7248caa833ef6cf3dd 100644 --- a/clang/test/Driver/linker-wrapper.c +++ b/clang/test/Driver/linker-wrapper.c @@ -123,8 +123,8 @@ // RUN: --linker-path=/usr/bin/ld --device-linker=a --device-linker=nvptx64-nvidia-cuda=b -- \ // RUN: %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=LINKER-ARGS -// LINKER-ARGS: clang{{.*}}--target=amdgcn-amd-amdhsa{{.*}}-Wl,a -// LINKER-ARGS: clang{{.*}}--target=nvptx64-nvidia-cuda{{.*}}-Wl,a -Wl,b +// LINKER-ARGS: clang{{.*}}--target=amdgcn-amd-amdhsa{{.*}}a +// LINKER-ARGS: clang{{.*}}--target=nvptx64-nvidia-cuda{{.*}}a b // RUN: not clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu -ldummy \ // RUN: --linker-path=/usr/bin/ld --device-linker=a --device-linker=nvptx64-nvidia-cuda=b -- \ diff --git a/clang/test/Driver/range.c b/clang/test/Driver/range.c new file mode 100644 index 0000000000000000000000000000000000000000..8d456a997d6967e3371a8ac7289d95a7c46deec8 --- /dev/null +++ b/clang/test/Driver/range.c @@ -0,0 +1,39 @@ +// Test range options for complex multiplication and division. + +// RUN: %clang -### -target x86_64 -fcx-limited-range -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=LMTD %s + +// RUN: %clang -### -target x86_64 -fno-cx-limited-range -c %s 2>&1 \ +// RUN: | FileCheck %s + +// RUN: %clang -### -target x86_64 -fcx-fortran-rules -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=FRTRN %s + +// RUN: %clang -### -target x86_64 -fno-cx-fortran-rules -c %s 2>&1 \ +// RUN: | FileCheck %s + +// RUN: %clang -### -target x86_64 -fcx-limited-range \ +// RUN: -fcx-fortran-rules -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=WARN1 %s + +// RUN: %clang -### -target x86_64 -fcx-fortran-rules \ +// RUN: -fcx-limited-range -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=WARN2 %s + +// RUN: %clang -### -target x86_64 -ffast-math -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=LMTD %s + +// RUN: %clang -### -target x86_64 -ffast-math -fcx-limited-range -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=LMTD %s + +// RUN: %clang -### -target x86_64 -fcx-limited-range -ffast-math -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=LMTD %s + +// LMTD: -complex-range=limited +// LMTD-NOT: -complex-range=fortran +// CHECK-NOT: -complex-range=limited +// FRTRN: -complex-range=fortran +// FRTRN-NOT: -complex-range=limited +// CHECK-NOT: -complex-range=fortran +// WARN1: warning: overriding '-fcx-limited-range' option with '-fcx-fortran-rules' [-Woverriding-option] +// WARN2: warning: overriding '-fcx-fortran-rules' option with '-fcx-limited-range' [-Woverriding-option] diff --git a/clang/test/Driver/rocm-detect.hip b/clang/test/Driver/rocm-detect.hip index 947c4f995be171522d049ed9e28a4c703ee21ba1..3644f215a345b6ea5d10eef8675656b138acdfa6 100644 --- a/clang/test/Driver/rocm-detect.hip +++ b/clang/test/Driver/rocm-detect.hip @@ -78,39 +78,39 @@ // RUN: | FileCheck -check-prefixes=ROCM-ENV,HIP-PATH %s // Test detecting latest /opt/rocm-{release} directory. -// RUN: rm -rf %T/opt -// RUN: mkdir -p %T/opt -// RUN: cp -r %S/Inputs/rocm %T/opt/rocm-3.9.0-1234 -// RUN: cp -r %S/Inputs/rocm %T/opt/rocm-3.10.0 -// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=gfx1010 --sysroot=%T \ +// RUN: rm -rf %t/opt +// RUN: mkdir -p %t/opt +// RUN: cp -r %S/Inputs/rocm %t/opt/rocm-3.9.0-1234 +// RUN: cp -r %S/Inputs/rocm %t/opt/rocm-3.10.0 +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=gfx1010 --sysroot=%t \ // RUN: --print-rocm-search-dirs %s 2>&1 \ // RUN: | FileCheck -check-prefixes=ROCM-REL %s -// Test ROCm installation built by SPACK by invoke clang at %T/rocm-spack/llvm-amdgpu-* +// Test ROCm installation built by SPACK by invoke clang at %t/rocm-spack/llvm-amdgpu-* // directory through a soft link. -// RUN: rm -rf %T/rocm-spack -// RUN: cp -r %S/Inputs/rocm-spack %T -// RUN: ln -fs %clang %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang -// RUN: %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang -### -v \ -// RUN: -resource-dir=%T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/lib/clang \ +// RUN: rm -rf %t/rocm-spack +// RUN: cp -r %S/Inputs/rocm-spack %t +// RUN: ln -fs %clang %t/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang +// RUN: %t/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang -### -v \ +// RUN: -resource-dir=%t/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/lib/clang \ // RUN: -target x86_64-linux-gnu --cuda-gpu-arch=gfx900 --print-rocm-search-dirs %s 2>&1 \ // RUN: | FileCheck -check-prefixes=SPACK %s // Test SPACK installation with multiple hip and rocm-device-libs packages of the same // ROCm release. --hip-path and --rocm-device-lib-path can be used to specify them. -// RUN: cp -r %T/rocm-spack/hip-* %T/rocm-spack/hip-4.0.0-abcd -// RUN: %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang -### -v \ +// RUN: cp -r %t/rocm-spack/hip-* %t/rocm-spack/hip-4.0.0-abcd +// RUN: %t/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang -### -v \ // RUN: -target x86_64-linux-gnu --cuda-gpu-arch=gfx900 \ -// RUN: --hip-path=%T/rocm-spack/hip-4.0.0-abcd \ +// RUN: --hip-path=%t/rocm-spack/hip-4.0.0-abcd \ // RUN: %s 2>&1 | FileCheck -check-prefixes=SPACK-SET %s // Test invalid SPACK ROCm installation missing hip and rocm-device-libs packages. -// RUN: rm -rf %T/rocm-spack/hip-* -// RUN: rm -rf %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/amdgcn -// RUN: %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang --version 2>&1 \ +// RUN: rm -rf %t/rocm-spack/hip-* +// RUN: rm -rf %t/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/amdgcn +// RUN: %t/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang --version 2>&1 \ // RUN: | FileCheck -check-prefixes=SPACK-MISS-SILENT %s // GFX902-DEFAULTLIBS: error: cannot find ROCm device library for gfx902; provide its path via '--rocm-path' or '--rocm-device-lib-path', or pass '-nogpulib' to build without ROCm device library diff --git a/clang/test/ExtractAPI/language.c b/clang/test/ExtractAPI/language.c index 6facd18f5d98106bf4740181d436898571f87461..fe98626c84613e0dfcfd6c655f5a3c2b28eeab7a 100644 --- a/clang/test/ExtractAPI/language.c +++ b/clang/test/ExtractAPI/language.c @@ -4,29 +4,42 @@ // RUN: %t/c.reference.output.json.in >> %t/c.reference.output.json // RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}@g" \ // RUN: %t/objc.reference.output.json.in >> %t/objc.reference.output.json +// RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}@g" \ +// RUN: %t/objcpp.reference.output.json.in >> %t/objcpp.reference.output.json -// RUN: %clang -extract-api -x c-header -target arm64-apple-macosx \ +// RUN: %clang_cc1 -extract-api -x c-header -triple arm64-apple-macosx \ // RUN: %t/c.h -o %t/c.output.json | FileCheck -allow-empty %s -// RUN: %clang -extract-api -x objective-c-header -target arm64-apple-macosx \ +// RUN: %clang_cc1 -extract-api -x objective-c-header -triple arm64-apple-macosx \ // RUN: %t/objc.h -o %t/objc.output.json | FileCheck -allow-empty %s +// RUN: %clang_cc1 -extract-api -x objective-c++-header -triple arm64-apple-macosx \ +// RUN: %t/objcpp.h -o %t/objcpp.output.json | FileCheck -allow-empty %s // Generator version is not consistent across test runs, normalize it. // RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \ // RUN: %t/c.output.json >> %t/c.output-normalized.json // RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \ // RUN: %t/objc.output.json >> %t/objc.output-normalized.json +// RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \ +// RUN: %t/objcpp.output.json >> %t/objcpp.output-normalized.json // RUN: diff %t/c.reference.output.json %t/c.output-normalized.json // RUN: diff %t/objc.reference.output.json %t/objc.output-normalized.json +// RUN: diff %t/objcpp.reference.output.json %t/objcpp.output-normalized.json // CHECK-NOT: error: // CHECK-NOT: warning: //--- c.h char c; +///expected-no-diagnostics //--- objc.h char objc; +///expected-no-diagnostics + +//--- objcpp.h +char objcpp; +///expected-no-diagnostics //--- c.reference.output.json.in { @@ -196,3 +209,87 @@ char objc; } ] } +//--- objcpp.reference.output.json.in +{ + "metadata": { + "formatVersion": { + "major": 0, + "minor": 5, + "patch": 3 + }, + "generator": "?" + }, + "module": { + "name": "", + "platform": { + "architecture": "arm64", + "operatingSystem": { + "minimumVersion": { + "major": 11, + "minor": 0, + "patch": 0 + }, + "name": "macosx" + }, + "vendor": "apple" + } + }, + "relationships": [], + "symbols": [ + { + "accessLevel": "public", + "declarationFragments": [ + { + "kind": "typeIdentifier", + "preciseIdentifier": "c:C", + "spelling": "char" + }, + { + "kind": "text", + "spelling": " " + }, + { + "kind": "identifier", + "spelling": "objcpp" + }, + { + "kind": "text", + "spelling": ";" + } + ], + "identifier": { + "interfaceLanguage": "objective-c++", + "precise": "c:@objcpp" + }, + "kind": { + "displayName": "Global Variable", + "identifier": "objective-c++.var" + }, + "location": { + "position": { + "character": 5, + "line": 0 + }, + "uri": "file://INPUT_DIR/objcpp.h" + }, + "names": { + "navigator": [ + { + "kind": "identifier", + "spelling": "objcpp" + } + ], + "subHeading": [ + { + "kind": "identifier", + "spelling": "objcpp" + } + ], + "title": "objcpp" + }, + "pathComponents": [ + "objcpp" + ] + } + ] +} diff --git a/clang/test/Layout/ms-x86-declspec-empty_bases.cpp b/clang/test/Layout/ms-x86-declspec-empty_bases.cpp index cc13a980cb5dbd3bbe18b1d6dfa36eb34ae4ca4f..4738ce5720f7546de51f50a6e2f5ec36b016f9b7 100644 --- a/clang/test/Layout/ms-x86-declspec-empty_bases.cpp +++ b/clang/test/Layout/ms-x86-declspec-empty_bases.cpp @@ -264,3 +264,63 @@ int _ = sizeof(G); // CHECK-NEXT: | [sizeof=12, align=4, // CHECK-NEXT: | nvsize=12, nvalign=4] } + +namespace test5 { + +struct A { + int a; +}; +struct B { + int b; +}; +struct C {}; +struct __declspec(align(16)) D {}; +struct E { + [[msvc::no_unique_address]] C c; +}; +struct __declspec(empty_bases) X : A, D, B, C, E { +}; + +// CHECK: *** Dumping AST Record Layout +// CHECK-NEXT: 0 | struct test5::A +// CHECK-NEXT: 0 | int a +// CHECK-NEXT: | [sizeof=4, align=4, +// CHECK-NEXT: | nvsize=4, nvalign=4] + +// CHECK: *** Dumping AST Record Layout +// CHECK-NEXT: 0 | struct test5::D (empty) +// CHECK-NEXT: | [sizeof=16, align=16, +// CHECK-NEXT: | nvsize=0, nvalign=16] + +// CHECK: *** Dumping AST Record Layout +// CHECK-NEXT: 0 | struct test5::B +// CHECK-NEXT: 0 | int b +// CHECK-NEXT: | [sizeof=4, align=4, +// CHECK-NEXT: | nvsize=4, nvalign=4] + +// CHECK: *** Dumping AST Record Layout +// CHECK-NEXT: 0 | struct test5::C (empty) +// CHECK-NEXT: | [sizeof=1, align=1, +// CHECK-NEXT: | nvsize=0, nvalign=1] + +// CHECK: *** Dumping AST Record Layout +// CHECK-NEXT: 0 | struct test5::E (empty) +// CHECK-NEXT: 0 | struct test5::C c (empty) +// CHECK-NEXT: | [sizeof=1, align=1, +// CHECK-NEXT: | nvsize=1, nvalign=1] + +// CHECK: *** Dumping AST Record Layout +// CHECK-NEXT: 0 | struct test5::X +// CHECK-NEXT: 0 | struct test5::A (base) +// CHECK-NEXT: 0 | int a +// CHECK-NEXT: 0 | struct test5::D (base) (empty) +// CHECK-NEXT: 0 | struct test5::C (base) (empty) +// CHECK-NEXT: 4 | struct test5::B (base) +// CHECK-NEXT: 4 | int b +// CHECK-NEXT: 8 | struct test5::E (base) (empty) +// CHECK-NEXT: 8 | struct test5::C c (empty) +// CHECK-NEXT: | [sizeof=16, align=16, +// CHECK-NEXT: | nvsize=16, nvalign=16] + +int _ = sizeof(X); +} diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test index 707fc8875089f7e483d82b406e656478181f19c4..bdfda430eea86c5ae5f4ff5a193ce225a29dabdf 100644 --- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test +++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test @@ -90,6 +90,7 @@ // CHECK-NEXT: LoaderUninitialized (SubjectMatchRule_variable_is_global) // CHECK-NEXT: Lockable (SubjectMatchRule_record) // CHECK-NEXT: MIGServerRoutine (SubjectMatchRule_function, SubjectMatchRule_objc_method, SubjectMatchRule_block) +// CHECK-NEXT: MSConstexpr (SubjectMatchRule_function) // CHECK-NEXT: MSStruct (SubjectMatchRule_record) // CHECK-NEXT: MaybeUndef (SubjectMatchRule_variable_is_parameter) // CHECK-NEXT: MicroMips (SubjectMatchRule_function) diff --git a/clang/test/Misc/target-invalid-cpu-note.c b/clang/test/Misc/target-invalid-cpu-note.c index c7146e63add5f20df7de098da665244083e9ea49..e840a9208f5a4507f988618008bd1d1abd275709 100644 --- a/clang/test/Misc/target-invalid-cpu-note.c +++ b/clang/test/Misc/target-invalid-cpu-note.c @@ -1,7 +1,7 @@ // Use CHECK-NEXT instead of multiple CHECK-SAME to ensure we will fail if there is anything extra in the output. // RUN: not %clang_cc1 -triple armv5--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix ARM // ARM: error: unknown target CPU 'not-a-cpu' -// ARM-NEXT: note: valid target CPU values are: arm8, arm810, strongarm, strongarm110, strongarm1100, strongarm1110, arm7tdmi, arm7tdmi-s, arm710t, arm720t, arm9, arm9tdmi, arm920, arm920t, arm922t, arm940t, ep9312, arm10tdmi, arm1020t, arm9e, arm946e-s, arm966e-s, arm968e-s, arm10e, arm1020e, arm1022e, arm926ej-s, arm1136j-s, arm1136jf-s, mpcore, mpcorenovfp, arm1176jz-s, arm1176jzf-s, arm1156t2-s, arm1156t2f-s, cortex-m0, cortex-m0plus, cortex-m1, sc000, cortex-a5, cortex-a7, cortex-a8, cortex-a9, cortex-a12, cortex-a15, cortex-a17, krait, cortex-r4, cortex-r4f, cortex-r5, cortex-r7, cortex-r8, cortex-r52, sc300, cortex-m3, cortex-m4, cortex-m7, cortex-m23, cortex-m33, cortex-m35p, cortex-m55, cortex-m85, cortex-a32, cortex-a35, cortex-a53, cortex-a55, cortex-a57, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78c, cortex-a710, cortex-x1, cortex-x1c, neoverse-n1, neoverse-n2, neoverse-v1, cyclone, exynos-m3, exynos-m4, exynos-m5, kryo, iwmmxt, xscale, swift{{$}} +// ARM-NEXT: note: valid target CPU values are: arm8, arm810, strongarm, strongarm110, strongarm1100, strongarm1110, arm7tdmi, arm7tdmi-s, arm710t, arm720t, arm9, arm9tdmi, arm920, arm920t, arm922t, arm940t, ep9312, arm10tdmi, arm1020t, arm9e, arm946e-s, arm966e-s, arm968e-s, arm10e, arm1020e, arm1022e, arm926ej-s, arm1136j-s, arm1136jf-s, mpcore, mpcorenovfp, arm1176jz-s, arm1176jzf-s, arm1156t2-s, arm1156t2f-s, cortex-m0, cortex-m0plus, cortex-m1, sc000, cortex-a5, cortex-a7, cortex-a8, cortex-a9, cortex-a12, cortex-a15, cortex-a17, krait, cortex-r4, cortex-r4f, cortex-r5, cortex-r7, cortex-r8, cortex-r52, sc300, cortex-m3, cortex-m4, cortex-m7, cortex-m23, cortex-m33, cortex-m35p, cortex-m55, cortex-m85, cortex-m52, cortex-a32, cortex-a35, cortex-a53, cortex-a55, cortex-a57, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78c, cortex-a710, cortex-x1, cortex-x1c, neoverse-n1, neoverse-n2, neoverse-v1, cyclone, exynos-m3, exynos-m4, exynos-m5, kryo, iwmmxt, xscale, swift{{$}} // RUN: not %clang_cc1 -triple arm64--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix AARCH64 // AARCH64: error: unknown target CPU 'not-a-cpu' @@ -29,7 +29,7 @@ // RUN: not %clang_cc1 -triple nvptx--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix NVPTX // NVPTX: error: unknown target CPU 'not-a-cpu' -// NVPTX-NEXT: note: valid target CPU values are: sm_20, sm_21, sm_30, sm_32, sm_35, sm_37, sm_50, sm_52, sm_53, sm_60, sm_61, sm_62, sm_70, sm_72, sm_75, sm_80, sm_86, sm_87, sm_89, sm_90, gfx600, gfx601, gfx602, gfx700, gfx701, gfx702, gfx703, gfx704, gfx705, gfx801, gfx802, gfx803, gfx805, gfx810, gfx900, gfx902, gfx904, gfx906, gfx908, gfx909, gfx90a, gfx90c, gfx940, gfx941, gfx942, gfx1010, gfx1011, gfx1012, gfx1013, gfx1030, gfx1031, gfx1032, gfx1033, gfx1034, gfx1035, gfx1036, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1200, gfx1201{{$}} +// NVPTX-NEXT: note: valid target CPU values are: sm_20, sm_21, sm_30, sm_32, sm_35, sm_37, sm_50, sm_52, sm_53, sm_60, sm_61, sm_62, sm_70, sm_72, sm_75, sm_80, sm_86, sm_87, sm_89, sm_90, sm_90a, gfx600, gfx601, gfx602, gfx700, gfx701, gfx702, gfx703, gfx704, gfx705, gfx801, gfx802, gfx803, gfx805, gfx810, gfx900, gfx902, gfx904, gfx906, gfx908, gfx909, gfx90a, gfx90c, gfx940, gfx941, gfx942, gfx1010, gfx1011, gfx1012, gfx1013, gfx1030, gfx1031, gfx1032, gfx1033, gfx1034, gfx1035, gfx1036, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1200, gfx1201{{$}} // RUN: not %clang_cc1 -triple r600--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix R600 // R600: error: unknown target CPU 'not-a-cpu' diff --git a/clang/test/Modules/decl-params-determinisim.m b/clang/test/Modules/decl-params-determinisim.m index 351403d9af947e2f43f5c1bfef0a78578b3287ef..9cf37ac4334cf103a14ce09902a629d976420c45 100644 --- a/clang/test/Modules/decl-params-determinisim.m +++ b/clang/test/Modules/decl-params-determinisim.m @@ -28,23 +28,23 @@ // CHECK: vint64m1_t test_vsmul_vv_i64m1(vint64m1_t op1, vint64m1_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m1(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m1(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vsmul_vx_i64m1(vint64m1_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m1(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m1(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vsmul_vv_i64m2(vint64m2_t op1, vint64m2_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m2(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m2(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vsmul_vx_i64m2(vint64m2_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m2(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m2(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vsmul_vv_i64m4(vint64m4_t op1, vint64m4_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m4(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m4(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vsmul_vx_i64m4(vint64m4_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m4(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m4(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vsmul_vv_i64m8(vint64m8_t op1, vint64m8_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m8(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m8(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vsmul_vx_i64m8(vint64m8_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m8(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m8(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vsmul_vv_i64m1_m(vbool64_t mask, vint64m1_t op1, vint64m1_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m1_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m1_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vsmul_vx_i64m1_m(vbool64_t mask, vint64m1_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m1_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m1_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vsmul_vv_i64m2_m(vbool32_t mask, vint64m2_t op1, vint64m2_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m2_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m2_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vsmul_vx_i64m2_m(vbool32_t mask, vint64m2_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m2_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m2_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vsmul_vv_i64m4_m(vbool16_t mask, vint64m4_t op1, vint64m4_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m4_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m4_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vsmul_vx_i64m4_m(vbool16_t mask, vint64m4_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m4_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m4_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vsmul_vv_i64m8_m(vbool8_t mask, vint64m8_t op1, vint64m8_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m8_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m8_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vsmul_vx_i64m8_m(vbool8_t mask, vint64m8_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m8_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m8_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulh_vv_i64m1(vint64m1_t op1, vint64m1_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m1(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m1(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulh_vx_i64m1(vint64m1_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m1(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m1(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulh_vv_i64m2(vint64m2_t op1, vint64m2_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m2(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m2(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulh_vx_i64m2(vint64m2_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m2(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m2(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulh_vv_i64m4(vint64m4_t op1, vint64m4_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m4(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m4(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulh_vx_i64m4(vint64m4_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m4(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m4(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulh_vv_i64m8(vint64m8_t op1, vint64m8_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m8(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m8(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulh_vx_i64m8(vint64m8_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m8(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m8(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulh_vv_i64m1_m(vbool64_t mask, vint64m1_t op1, vint64m1_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulh_vx_i64m1_m(vbool64_t mask, vint64m1_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulh_vv_i64m2_m(vbool32_t mask, vint64m2_t op1, vint64m2_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulh_vx_i64m2_m(vbool32_t mask, vint64m2_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulh_vv_i64m4_m(vbool16_t mask, vint64m4_t op1, vint64m4_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulh_vx_i64m4_m(vbool16_t mask, vint64m4_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulh_vv_i64m8_m(vbool8_t mask, vint64m8_t op1, vint64m8_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulh_vx_i64m8_m(vbool8_t mask, vint64m8_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m1_t test_vmulhu_vv_u64m1(vuint64m1_t op1, vuint64m1_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m1(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m1(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m1_t test_vmulhu_vx_u64m1(vuint64m1_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m1(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m1(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m2_t test_vmulhu_vv_u64m2(vuint64m2_t op1, vuint64m2_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m2(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m2(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m2_t test_vmulhu_vx_u64m2(vuint64m2_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m2(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m2(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m4_t test_vmulhu_vv_u64m4(vuint64m4_t op1, vuint64m4_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m4(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m4(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m4_t test_vmulhu_vx_u64m4(vuint64m4_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m4(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m4(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m8_t test_vmulhu_vv_u64m8(vuint64m8_t op1, vuint64m8_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m8(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m8(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m8_t test_vmulhu_vx_u64m8(vuint64m8_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m8(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m8(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m1_t test_vmulhu_vv_u64m1_m(vbool64_t mask, vuint64m1_t op1, vuint64m1_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m1_t test_vmulhu_vx_u64m1_m(vbool64_t mask, vuint64m1_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m2_t test_vmulhu_vv_u64m2_m(vbool32_t mask, vuint64m2_t op1, vuint64m2_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m2_t test_vmulhu_vx_u64m2_m(vbool32_t mask, vuint64m2_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m4_t test_vmulhu_vv_u64m4_m(vbool16_t mask, vuint64m4_t op1, vuint64m4_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m4_t test_vmulhu_vx_u64m4_m(vbool16_t mask, vuint64m4_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m8_t test_vmulhu_vv_u64m8_m(vbool8_t mask, vuint64m8_t op1, vuint64m8_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m8_t test_vmulhu_vx_u64m8_m(vbool8_t mask, vuint64m8_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulhsu_vv_i64m1(vint64m1_t op1, vuint64m1_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m1(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m1(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulhsu_vx_i64m1(vint64m1_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m1(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m1(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulhsu_vv_i64m2(vint64m2_t op1, vuint64m2_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m2(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m2(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulhsu_vx_i64m2(vint64m2_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m2(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m2(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulhsu_vv_i64m4(vint64m4_t op1, vuint64m4_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m4(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m4(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulhsu_vx_i64m4(vint64m4_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m4(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m4(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulhsu_vv_i64m8(vint64m8_t op1, vuint64m8_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m8(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m8(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulhsu_vx_i64m8(vint64m8_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m8(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m8(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulhsu_vv_i64m1_m(vbool64_t mask, vint64m1_t op1, vuint64m1_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulhsu_vx_i64m1_m(vbool64_t mask, vint64m1_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulhsu_vv_i64m2_m(vbool32_t mask, vint64m2_t op1, vuint64m2_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulhsu_vx_i64m2_m(vbool32_t mask, vint64m2_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulhsu_vv_i64m4_m(vbool16_t mask, vint64m4_t op1, vuint64m4_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulhsu_vx_i64m4_m(vbool16_t mask, vint64m4_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulhsu_vv_i64m8_m(vbool8_t mask, vint64m8_t op1, vuint64m8_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulhsu_vx_i64m8_m(vbool8_t mask, vint64m8_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } diff --git a/clang/test/Sema/scoped-atomic-ops.c b/clang/test/Sema/scoped-atomic-ops.c new file mode 100644 index 0000000000000000000000000000000000000000..59e638c646664c309998611d307f941f46db4c64 --- /dev/null +++ b/clang/test/Sema/scoped-atomic-ops.c @@ -0,0 +1,101 @@ +// RUN: %clang_cc1 -x c -triple=amdgcn-amd-amdhsa -verify -fsyntax-only %s +// RUN: %clang_cc1 -x c -triple=x86_64-pc-linux-gnu -verify -fsyntax-only %s + +int fi1a(int *i) { + int v; + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return v; +} + +int fi1b(int *i) { + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 3, have 2}} + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return *i; +} + +int fi2a(int *i) { + int v; + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return v; +} + +void fi2b(int *i) { + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); +} + +void fi3a(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); +} + +void fi3b(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *b = __scoped_atomic_fetch_sub(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *c = __scoped_atomic_fetch_and(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *d = __scoped_atomic_fetch_or(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *e = __scoped_atomic_fetch_xor(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *f = __scoped_atomic_fetch_nand(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *g = __scoped_atomic_fetch_min(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *h = __scoped_atomic_fetch_max(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} +} + +void fi3c(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} +} + +void fi3d(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} +} + +int fi4a(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SYSTEM); +} + +int fi5a(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n(i, &cmp, 1, 1, __ATOMIC_ACQUIRE, + __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SYSTEM); +} + +int fi6a(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return ret; +} + +int fi7a(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_SYSTEM); +} diff --git a/clang/test/Sema/switch-default.c b/clang/test/Sema/switch-default.c new file mode 100644 index 0000000000000000000000000000000000000000..854b561b37c48e4e2d12ec7737a8509fa742ba19 --- /dev/null +++ b/clang/test/Sema/switch-default.c @@ -0,0 +1,17 @@ +// RUN: %clang_cc1 -fsyntax-only -verify -Wswitch-default %s + +int f1(int a) { + switch (a) { // expected-warning {{'switch' missing 'default' label}} + case 1: a++; break; + case 2: a += 2; break; + } + return a; +} + +int f2(int a) { + switch (a) { // no-warning + default: + ; + } + return a; +} diff --git a/clang/test/SemaCXX/coro-lifetimebound.cpp b/clang/test/SemaCXX/coro-lifetimebound.cpp index d3e2d673ebb3c0d4ff949a48be265a40aeeb42df..b4dc029a13984846f54447eaed9ef327059f7e3c 100644 --- a/clang/test/SemaCXX/coro-lifetimebound.cpp +++ b/clang/test/SemaCXX/coro-lifetimebound.cpp @@ -1,4 +1,4 @@ -// RUN: %clang_cc1 -triple x86_64-apple-darwin9 %s -std=c++20 -fsyntax-only -verify -Wall -Wextra -Wno-error=unreachable-code -Wno-unused +// RUN: %clang_cc1 -triple x86_64-apple-darwin9 %s -std=c++20 -fsyntax-only -verify -Wall -Wextra -Wno-error=unreachable-code -Wno-unused -Wno-c++23-lambda-attributes #include "Inputs/std-coroutine.h" @@ -64,14 +64,8 @@ Co bar_coro(const int &b, int c) { : bar_coro(0, 1); // expected-warning {{returning address of local temporary object}} } -#define CORO_WRAPPER \ - _Pragma("clang diagnostic push") \ - _Pragma("clang diagnostic ignored \"-Wc++23-extensions\"") \ - [[clang::coro_wrapper]] \ - _Pragma("clang diagnostic pop") - void lambdas() { - auto unsafe_lambda = [] CORO_WRAPPER (int b) { + auto unsafe_lambda = [] [[clang::coro_wrapper]] (int b) { return foo_coro(b); // expected-warning {{address of stack memory associated with parameter}} }; auto coro_lambda = [] (const int&) -> Co { diff --git a/clang/test/SemaCXX/coro-return-type-and-wrapper.cpp b/clang/test/SemaCXX/coro-return-type-and-wrapper.cpp index 5f8076f1c782ac3ca7213580104d81aac00179a3..ac49e03ba9d90ad2d670fbe5c72f11f2443847c5 100644 --- a/clang/test/SemaCXX/coro-return-type-and-wrapper.cpp +++ b/clang/test/SemaCXX/coro-return-type-and-wrapper.cpp @@ -1,4 +1,4 @@ -// RUN: %clang_cc1 -triple x86_64-apple-darwin9 %s -std=c++20 -fsyntax-only -verify -Wall -Wextra +// RUN: %clang_cc1 -triple x86_64-apple-darwin9 %s -std=c++20 -fsyntax-only -verify -Wall -Wextra -Wno-c++23-lambda-attributes #include "Inputs/std-coroutine.h" using std::suspend_always; @@ -45,11 +45,6 @@ Co non_marked_wrapper(int b) { return foo_coro(b); } } // namespace using_decl namespace lambdas { -#define CORO_WRAPPER \ - _Pragma("clang diagnostic push") \ - _Pragma("clang diagnostic ignored \"-Wc++23-extensions\"") \ - [[clang::coro_wrapper]] \ - _Pragma("clang diagnostic pop") void foo() { auto coro_lambda = []() -> Gen { @@ -59,7 +54,7 @@ void foo() { auto not_allowed_wrapper = []() -> Gen { return foo_coro(1); }; - auto allowed_wrapper = [] CORO_WRAPPER() -> Gen { + auto allowed_wrapper = [] [[clang::coro_wrapper]] () -> Gen { return foo_coro(1); }; } diff --git a/clang/test/SemaCXX/cxx1z-copy-omission.cpp b/clang/test/SemaCXX/cxx1z-copy-omission.cpp index a850cf6143cd48057f0a0eaaed05eb332af9e52a..f46a17af833867737a297c0d7d4a08bf27eb0179 100644 --- a/clang/test/SemaCXX/cxx1z-copy-omission.cpp +++ b/clang/test/SemaCXX/cxx1z-copy-omission.cpp @@ -171,3 +171,30 @@ namespace CtorTemplateBeatsNonTemplateConversionFn { Foo f(Derived d) { return d; } // expected-error {{invokes a deleted function}} Foo g(Derived d) { return Foo(d); } // ok, calls constructor } + +// Make sure we don't consider conversion functions for guaranteed copy elision +namespace GH39319 { +struct A { + A(); + A(const A&) = delete; // expected-note {{'A' has been explicitly marked deleted here}} +}; +struct B { + operator A(); +} C; +A::A() : A(C) {} // expected-error {{call to deleted constructor of}} + +struct A2 { + struct B2 { + operator A2(); + }; + A2() : A2(B2()) {} // expected-error {{call to deleted constructor of}} + A2(const A2&) = delete; // expected-note {{'A2' has been explicitly marked deleted here}} +}; + +template +class B3 : A3 { + template()> // expected-warning 2{{use of function template name with no prior declaration in function call with explicit}} + B3(); +}; B3(); // expected-error {{deduction guide declaration without trailing return type}} \ + // expected-note {{while building implicit deduction guide first needed here}} +} diff --git a/clang/test/SemaCXX/friend.cpp b/clang/test/SemaCXX/friend.cpp index 367d6a6c1807c92049a5ae9d85154635a8e89de8..53e6bbfcf42a8ed8bc898bc4a7674c8fbb70ae4b 100644 --- a/clang/test/SemaCXX/friend.cpp +++ b/clang/test/SemaCXX/friend.cpp @@ -162,7 +162,7 @@ namespace test9 { class C { }; struct A { - friend void C::f(int, int, int) {} // expected-error {{friend function definition cannot be qualified with 'C::'}} + friend void C::f(int, int, int) {} // expected-error {{friend declaration of 'f' does not match any declaration in 'test9::C'}} }; } diff --git a/clang/test/SemaCXX/ms-constexpr-invalid.cpp b/clang/test/SemaCXX/ms-constexpr-invalid.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e5bec0c7119b02871681e5a8465ce7815689af46 --- /dev/null +++ b/clang/test/SemaCXX/ms-constexpr-invalid.cpp @@ -0,0 +1,52 @@ +// RUN: %clang_cc1 -fms-compatibility -fms-compatibility-version=19.33 -std=c++20 -verify %s +// RUN: %clang_cc1 -fms-compatibility -fms-compatibility-version=19.33 -std=c++17 -verify %s + +// Check explicitly invalid code + +void runtime() {} // expected-note {{declared here}} + +[[msvc::constexpr]] void f0() { runtime(); } // expected-error {{constexpr function never produces a constant expression}} \ + // expected-note {{non-constexpr function 'runtime' cannot be used in a constant expression}} +[[msvc::constexpr]] constexpr void f1() {} // expected-error {{attribute 'msvc::constexpr' cannot be applied to the constexpr function 'f1'}} +#if __cplusplus >= 202202L +[[msvc::constexpr]] consteval void f2() {} // expected-error {{attribute 'msvc::constexpr' cannot be applied to the consteval function 'f1'}} +#endif + +struct B1 {}; +struct D1 : virtual B1 { // expected-note {{virtual base class declared here}} + [[msvc::constexpr]] D1() {} // expected-error {{constexpr constructor not allowed in struct with virtual base class}} +}; + +struct [[msvc::constexpr]] S2{}; // expected-error {{'constexpr' attribute only applies to functions and return statements}} + +// Check invalid code mixed with valid code + +[[msvc::constexpr]] int f4(int x) { return x > 1 ? 1 + f4(x / 2) : 0; } // expected-note {{non-constexpr function 'f4' cannot be used in a constant expression}} \ + // expected-note {{declared here}} \ + // expected-note {{declared here}} \ + // expected-note {{declared here}} +constexpr bool f5() { [[msvc::constexpr]] return f4(32) == 5; } // expected-note {{in call to 'f4(32)'}} +static_assert(f5()); // expected-error {{static assertion expression is not an integral constant expression}} \ + // expected-note {{in call to 'f5()'}} + +int f6(int x) { [[msvc::constexpr]] return x > 1 ? 1 + f6(x / 2) : 0; } // expected-note {{declared here}} \ + // expected-note {{declared here}} +constexpr bool f7() { [[msvc::constexpr]] return f6(32) == 5; } // expected-error {{constexpr function never produces a constant expression}} \ + // expected-note {{non-constexpr function 'f6' cannot be used in a constant expression}} \ + // expected-note {{non-constexpr function 'f6' cannot be used in a constant expression}} +static_assert(f7()); // expected-error {{static assertion expression is not an integral constant expression}} \ + // expected-note {{in call to 'f7()'}} + +constexpr bool f8() { // expected-error {{constexpr function never produces a constant expression}} + [[msvc::constexpr]] f4(32); // expected-error {{'constexpr' attribute only applies to functions and return statements}} \ + // expected-note {{non-constexpr function 'f4' cannot be used in a constant expression}} \ + // expected-note {{non-constexpr function 'f4' cannot be used in a constant expression}} + [[msvc::constexpr]] int i5 = f4(32); // expected-error {{'constexpr' attribute only applies to functions and return statements}} + return i5 == 5; +} +static_assert(f8()); // expected-error {{static assertion expression is not an integral constant expression}} \ + // expected-note {{in call to 'f8()'}} + +#if __cplusplus == 201702L +struct S1 { [[msvc::constexpr]] virtual bool vm() const { return true; } }; // expected-error {{attribute 'msvc::constexpr' ignored, it only applies to function definitions and return statements}} +#endif diff --git a/clang/test/SemaCXX/ms-constexpr-new.cpp b/clang/test/SemaCXX/ms-constexpr-new.cpp new file mode 100644 index 0000000000000000000000000000000000000000..30567740b2ecbbe383f4ce4c0f70d6f9f29148b7 --- /dev/null +++ b/clang/test/SemaCXX/ms-constexpr-new.cpp @@ -0,0 +1,16 @@ +// RUN: %clang_cc1 -fms-compatibility -fms-compatibility-version=19.33 -std=c++20 -verify=supported %s +// RUN: %clang_cc1 -fms-compatibility -fms-compatibility-version=19.32 -std=c++20 -verify=unsupported %s +// supported-no-diagnostics + +[[nodiscard]] +[[msvc::constexpr]] // unsupported-warning {{unknown attribute 'constexpr' ignored}} +inline void* operator new(decltype(sizeof(void*)), void* p) noexcept { return p; } + +namespace std { + constexpr int* construct_at(int* p, int v) { + [[msvc::constexpr]] return ::new (p) int(v); // unsupported-warning {{unknown attribute 'constexpr' ignored}} + } +} + +constexpr bool check_construct_at() { int x; return *std::construct_at(&x, 42) == 42; } +static_assert(check_construct_at()); diff --git a/clang/test/SemaCXX/ms-constexpr.cpp b/clang/test/SemaCXX/ms-constexpr.cpp new file mode 100644 index 0000000000000000000000000000000000000000..79f71a34cb7d8485b1f95cdd34c8798bfa7e0f4b --- /dev/null +++ b/clang/test/SemaCXX/ms-constexpr.cpp @@ -0,0 +1,37 @@ +// RUN: %clang_cc1 -fms-compatibility -fms-compatibility-version=19.33 -std=c++20 -verify %s + +[[msvc::constexpr]] int log2(int x) { [[msvc::constexpr]] return x > 1 ? 1 + log2(x / 2) : 0; } +constexpr bool test_log2() { [[msvc::constexpr]] return log2(32) == 5; } +static_assert(test_log2()); + +[[msvc::constexpr]] int get_value(int x) +{ + switch (x) + { + case 42: return 1337; + default: + if (x < 0) [[msvc::constexpr]] return log2(-x); + else return x; + } +} + +constexpr bool test_complex_expr() { + [[msvc::constexpr]] return get_value(get_value(42) - 1337 + get_value(-32) - 5 + (get_value(1) ? get_value(0) : get_value(2))) == get_value(0); +} +static_assert(test_complex_expr()); + +constexpr bool get_constexpr_true() { return true; } +[[msvc::constexpr]] bool get_msconstexpr_true() { return get_constexpr_true(); } +constexpr bool test_get_msconstexpr_true() { [[msvc::constexpr]] return get_msconstexpr_true(); } +static_assert(test_get_msconstexpr_true()); + +// TODO (#72149): Add support for [[msvc::constexpr]] constructor; this code is valid for MSVC. +struct S2 { + [[msvc::constexpr]] S2() {} + [[msvc::constexpr]] bool value() { return true; } + static constexpr bool check() { [[msvc::constexpr]] return S2{}.value(); } // expected-error {{constexpr function never produces a constant expression}} \ + // expected-note {{non-literal type 'S2' cannot be used in a constant expression}} \ + // expected-note {{non-literal type 'S2' cannot be used in a constant expression}} +}; +static_assert(S2::check()); // expected-error {{static assertion expression is not an integral constant expression}} \ + // expected-note {{in call to 'check()'}} diff --git a/clang/test/SemaCXX/warn-thread-safety-analysis.cpp b/clang/test/SemaCXX/warn-thread-safety-analysis.cpp index 205cfa284f6c9c96c4db5e811050d62b5afd8666..dfb966d3b5902d6bd2564cc7bdf2772089cf552b 100644 --- a/clang/test/SemaCXX/warn-thread-safety-analysis.cpp +++ b/clang/test/SemaCXX/warn-thread-safety-analysis.cpp @@ -1702,6 +1702,8 @@ struct TestScopedLockable { bool getBool(); + bool lock2Bool(MutexLock); + void foo1() { MutexLock mulock(&mu1); a = 5; @@ -1718,6 +1720,12 @@ struct TestScopedLockable { MutexLock{&mu1}, a = 5; } + void temporary_cfg(int x) { + // test the case where a pair of temporary Ctor and Dtor is in different CFG blocks + lock2Bool(MutexLock{&mu1}) || x; + MutexLock{&mu1}; // no-warn + } + void lifetime_extension() { const MutexLock &mulock = MutexLock(&mu1); a = 5; diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-fixits-add-assign.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-fixits-add-assign.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5c03cc10025c684b563ee4ea43d67dd2da0f21ba --- /dev/null +++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-fixits-add-assign.cpp @@ -0,0 +1,59 @@ +// RUN: %clang_cc1 -std=c++20 -Wunsafe-buffer-usage \ +// RUN: -fsafe-buffer-usage-suggestions \ +// RUN: -fdiagnostics-parseable-fixits %s 2>&1 | FileCheck %s +void foo(int * , int *); + +void add_assign_test(unsigned int n, int *a, int y) { + int *p = new int[10]; + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:3-[[@LINE-1]]:11}:"std::span p" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:12-[[@LINE-2]]:12}:"{" + // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:23-[[@LINE-3]]:23}:", 10}" + p += 2; + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:3-[[@LINE-1]]:7}:"p = p.subspan(" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:9-[[@LINE-2]]:9}:")" + + int *r = p; + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:3-[[@LINE-1]]:11}:"std::span r" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:12-[[@LINE-2]]:12}:"{" + // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:13-[[@LINE-3]]:13}:", <# placeholder #>}" + while (*r != 0) { + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:10-[[@LINE-1]]:11}:"" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:12-[[@LINE-2]]:12}:"[0]" + r += 2; + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:5-[[@LINE-1]]:9}:"r = r.subspan(" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:11-[[@LINE-2]]:11}:")" + } + + if (*p == 0) { + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:7-[[@LINE-1]]:8}:"" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:9-[[@LINE-2]]:9}:"[0]" + p += n; + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:5-[[@LINE-1]]:9}:"p = p.subspan(" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:11-[[@LINE-2]]:11}:")" + } + + if (*p == 1) + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:7-[[@LINE-1]]:8}:"" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:9-[[@LINE-2]]:9}:"[0]" + p += 3; + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:5-[[@LINE-1]]:9}:"p = p.subspan(" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:11-[[@LINE-2]]:11}:")" + + a += -9; + // CHECK-NOT: fix-it:"{{.*}}":{[[@LINE-1]]:5-[[@LINE-1]]:9}:"p = p.subspan(" + + a += y; + // CHECK-NOT: fix-it:"{{.*}}":{[[@LINE-1]]:5-[[@LINE-1]]:9}:"p = p.subspan(" +} + +int expr_test(unsigned x, int *q, int y) { + char *p = new char[8]; + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:3-[[@LINE-1]]:12}:"std::span p" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:13-[[@LINE-2]]:13}:"{" + // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:24-[[@LINE-3]]:24}:", 8}" + p += (x + 1); + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:3-[[@LINE-1]]:7}:"p = p.subspan" + + q += (y + 7); + // CHECK-NOT: fix-it:"{{.*}}":{[[@LINE-1]]:3-[[@LINE-1]]:7}:"q = q.subspan" +} diff --git a/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl b/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl index a9f6567aa00b8c3ed2c6246b9a3711af0799ad40..7e79ae3bf005fccef670103b4f18622195468a7b 100644 --- a/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl @@ -5,6 +5,14 @@ typedef vector float3; RWBuffer Buffer; +// expected-error@+2 {{class template 'RWBuffer' requires template arguments}} +// expected-note@*:* {{template declaration from hidden source: template class RWBuffer final}} +RWBuffer BufferErr1; + +// expected-error@+2 {{too few template arguments for class template 'RWBuffer'}} +// expected-note@*:* {{template declaration from hidden source: template class RWBuffer final}} +RWBuffer<> BufferErr2; + [numthreads(1,1,1)] void main() { (void)Buffer.h; // expected-error {{'h' is a private member of 'hlsl::RWBuffer'}} diff --git a/clang/test/SemaHLSL/BuiltIns/vector-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/vector-errors.hlsl index d2598a57400b0451e84ce7463e3ea69b232c8b0a..6aedb9304ed9339a0f54e6aa9afba13524c91b9c 100644 --- a/clang/test/SemaHLSL/BuiltIns/vector-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/vector-errors.hlsl @@ -2,17 +2,16 @@ // Some bad declarations hlsl::vector ShouldWorkSomeday; // expected-error{{use of alias template 'hlsl::vector' requires template arguments}} +// expected-note@*:* {{template declaration from hidden source: template using vector = element __attribute__((ext_vector_type(element_count)))}} hlsl::vector<1> BadVec; // expected-error{{template argument for template type parameter must be a type}} -// expected-note@*:* {{template is declared here}} -// expected-note@*:* {{template parameter is declared here}} - +// expected-note@*:* {{template parameter from hidden source: class element = float}} hlsl::vector AnotherBadVec; // expected-error{{template argument for non-type template parameter must be an expression}} -// expected-note@*:* {{template parameter is declared here}} +// expected-note@*:* {{template parameter from hidden source: int element_count = 4}} hlsl::vector YABV; // expected-error{{too many template arguments for alias template 'vector'}} -// expected-note@*:* {{template is declared here}} +// expected-note@*:* {{template declaration from hidden source: template using vector = element __attribute__((ext_vector_type(element_count)))}} // This code is rejected by clang because clang puts the HLSL built-in types // into the HLSL namespace. diff --git a/clang/test/SemaTemplate/GH71595.cpp b/clang/test/SemaTemplate/GH71595.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7d34d1bf054e4bc49de6b0abb4691a4763e64126 --- /dev/null +++ b/clang/test/SemaTemplate/GH71595.cpp @@ -0,0 +1,34 @@ +// RUN: %clang_cc1 -std=c++20 -verify %s + +template +concept C = true; + +class non_temp { + template T> + friend void f(); + + non_temp(); +}; + +template T> +void f() { + auto v = non_temp(); +} + +template +class temp { + template T> + friend void g(); + + temp(); // expected-note {{implicitly declared private here}} +}; + +template> T> +void g() { + auto v = temp(); // expected-error {{calling a private constructor of class 'temp'}} +} + +void h() { + f(); + g(); // expected-note {{in instantiation of function template specialization 'g' requested here}} +} diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp index 829f85b93bc73cbb1577d35137a0acaaa8ea5ef6..d2e3d8d43aef21a2f10bf094e0042138374fa633 100644 --- a/clang/tools/clang-format/ClangFormat.cpp +++ b/clang/tools/clang-format/ClangFormat.cpp @@ -398,8 +398,8 @@ class ClangFormatDiagConsumer : public DiagnosticConsumer { }; // Returns true on error. -static bool format(StringRef FileName) { - if (!OutputXML && Inplace && FileName == "-") { +static bool format(StringRef FileName, bool IsSTDIN) { + if (!OutputXML && Inplace && IsSTDIN) { errs() << "error: cannot use -i when reading from stdin.\n"; return false; } @@ -423,7 +423,7 @@ static bool format(StringRef FileName) { if (InvalidBOM) { errs() << "error: encoding with unsupported byte order mark \"" << InvalidBOM << "\" detected"; - if (FileName != "-") + if (!IsSTDIN) errs() << " in file '" << FileName << "'"; errs() << ".\n"; return true; @@ -432,7 +432,7 @@ static bool format(StringRef FileName) { std::vector Ranges; if (fillRanges(Code.get(), Ranges)) return true; - StringRef AssumedFileName = (FileName == "-") ? AssumeFileName : FileName; + StringRef AssumedFileName = IsSTDIN ? AssumeFileName : FileName; if (AssumedFileName.empty()) { llvm::errs() << "error: empty filenames are not allowed\n"; return true; @@ -544,28 +544,23 @@ static void PrintVersion(raw_ostream &OS) { } // Dump the configuration. -static int dumpConfig() { - StringRef FileName; +static int dumpConfig(bool IsSTDIN) { std::unique_ptr Code; - if (FileNames.empty()) { - // We can't read the code to detect the language if there's no - // file name, so leave Code empty here. - FileName = AssumeFileName; - } else { - // Read in the code in case the filename alone isn't enough to - // detect the language. + // We can't read the code to detect the language if there's no file name. + if (!IsSTDIN) { + // Read in the code in case the filename alone isn't enough to detect the + // language. ErrorOr> CodeOrErr = MemoryBuffer::getFileOrSTDIN(FileNames[0]); if (std::error_code EC = CodeOrErr.getError()) { llvm::errs() << EC.message() << "\n"; return 1; } - FileName = (FileNames[0] == "-") ? AssumeFileName : FileNames[0]; Code = std::move(CodeOrErr.get()); } llvm::Expected FormatStyle = - clang::format::getStyle(Style, FileName, FallbackStyle, - Code ? Code->getBuffer() : ""); + clang::format::getStyle(Style, IsSTDIN ? AssumeFileName : FileNames[0], + FallbackStyle, Code ? Code->getBuffer() : ""); if (!FormatStyle) { llvm::errs() << llvm::toString(FormatStyle.takeError()) << "\n"; return 1; @@ -596,8 +591,11 @@ int main(int argc, const char **argv) { return 0; } + if (FileNames.empty()) + FileNames.push_back("-"); + if (DumpConfig) - return dumpConfig(); + return dumpConfig(FileNames[0] == "-"); if (!Files.empty()) { std::ifstream ExternalFileOfFiles{std::string(Files)}; @@ -610,11 +608,6 @@ int main(int argc, const char **argv) { errs() << "Clang-formating " << LineNo << " files\n"; } - bool Error = false; - if (FileNames.empty()) { - Error = clang::format::format("-"); - return Error ? 1 : 0; - } if (FileNames.size() != 1 && (!Offsets.empty() || !Lengths.empty() || !LineRanges.empty())) { errs() << "error: -offset, -length and -lines can only be used for " @@ -623,12 +616,13 @@ int main(int argc, const char **argv) { } unsigned FileNo = 1; + bool Error = false; for (const auto &FileName : FileNames) { if (Verbose) { errs() << "Formatting [" << FileNo++ << "/" << FileNames.size() << "] " << FileName << "\n"; } - Error |= clang::format::format(FileName); + Error |= clang::format::format(FileName, FileName == "-"); } return Error ? 1 : 0; } diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp index db0ce3e2a1901922aa31640eafd947b96492bf36..5d2fe98fe56011588a4fa21748cd2c9bc10565a8 100644 --- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp +++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp @@ -428,7 +428,7 @@ Expected clang(ArrayRef InputFiles, const ArgList &Args) { std::back_inserter(CmdArgs)); for (StringRef Arg : Args.getAllArgValues(OPT_linker_arg_EQ)) - CmdArgs.push_back(Args.MakeArgString("-Wl," + Arg)); + CmdArgs.push_back(Args.MakeArgString(Arg)); for (StringRef Arg : Args.getAllArgValues(OPT_builtin_bitcode_EQ)) { if (llvm::Triple(Arg.split('=').first) == Triple) diff --git a/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp b/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp index 3e1dd874216ccf3b04053a1be14a78a71e105a0c..f4f500b173572d2e62f64451059c1ae1b5414b43 100644 --- a/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp +++ b/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp @@ -26,19 +26,6 @@ namespace { constexpr unsigned CudaFatMagic = 0x466243b1; constexpr unsigned HIPFatMagic = 0x48495046; -/// Copied from clang/CGCudaRuntime.h. -enum OffloadEntryKindFlag : uint32_t { - /// Mark the entry as a global entry. This indicates the presense of a - /// kernel if the size size field is zero and a variable otherwise. - OffloadGlobalEntry = 0x0, - /// Mark the entry as a managed global variable. - OffloadGlobalManagedEntry = 0x1, - /// Mark the entry as a surface variable. - OffloadGlobalSurfaceEntry = 0x2, - /// Mark the entry as a texture variable. - OffloadGlobalTextureEntry = 0x3, -}; - IntegerType *getSizeTTy(Module &M) { return M.getDataLayout().getIntPtrType(M.getContext()); } @@ -333,6 +320,24 @@ Function *createRegisterGlobalsFunction(Module &M, bool IsHIP) { FunctionCallee RegVar = M.getOrInsertFunction( IsHIP ? "__hipRegisterVar" : "__cudaRegisterVar", RegVarTy); + // Get the __cudaRegisterSurface function declaration. + auto *RegSurfaceTy = + FunctionType::get(Type::getVoidTy(C), + {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, + Type::getInt32Ty(C), Type::getInt32Ty(C)}, + /*isVarArg=*/false); + FunctionCallee RegSurface = M.getOrInsertFunction( + IsHIP ? "__hipRegisterSurface" : "__cudaRegisterSurface", RegSurfaceTy); + + // Get the __cudaRegisterTexture function declaration. + auto *RegTextureTy = FunctionType::get( + Type::getVoidTy(C), + {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C), + Type::getInt32Ty(C), Type::getInt32Ty(C)}, + /*isVarArg=*/false); + FunctionCallee RegTexture = M.getOrInsertFunction( + IsHIP ? "__hipRegisterTexture" : "__cudaRegisterTexture", RegTextureTy); + auto *RegGlobalsTy = FunctionType::get(Type::getVoidTy(C), Int8PtrPtrTy, /*isVarArg*/ false); auto *RegGlobalsFn = @@ -375,7 +380,31 @@ Function *createRegisterGlobalsFunction(Module &M, bool IsHIP) { Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry, {ConstantInt::get(getSizeTTy(M), 0), ConstantInt::get(Type::getInt32Ty(C), 3)}); - auto *Flags = Builder.CreateLoad(Type::getInt32Ty(C), FlagsPtr, "flag"); + auto *Flags = Builder.CreateLoad(Type::getInt32Ty(C), FlagsPtr, "flags"); + auto *DataPtr = + Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry, + {ConstantInt::get(getSizeTTy(M), 0), + ConstantInt::get(Type::getInt32Ty(C), 4)}); + auto *Data = Builder.CreateLoad(Type::getInt32Ty(C), DataPtr, "textype"); + auto *Kind = Builder.CreateAnd( + Flags, ConstantInt::get(Type::getInt32Ty(C), 0x7), "type"); + + // Extract the flags stored in the bit-field and convert them to C booleans. + auto *ExternBit = Builder.CreateAnd( + Flags, ConstantInt::get(Type::getInt32Ty(C), + llvm::offloading::OffloadGlobalExtern)); + auto *Extern = Builder.CreateLShr( + ExternBit, ConstantInt::get(Type::getInt32Ty(C), 3), "extern"); + auto *ConstantBit = Builder.CreateAnd( + Flags, ConstantInt::get(Type::getInt32Ty(C), + llvm::offloading::OffloadGlobalConstant)); + auto *Const = Builder.CreateLShr( + ConstantBit, ConstantInt::get(Type::getInt32Ty(C), 4), "constant"); + auto *NormalizedBit = Builder.CreateAnd( + Flags, ConstantInt::get(Type::getInt32Ty(C), + llvm::offloading::OffloadGlobalNormalized)); + auto *Normalized = Builder.CreateLShr( + NormalizedBit, ConstantInt::get(Type::getInt32Ty(C), 5), "normalized"); auto *FnCond = Builder.CreateICmpEQ(Size, ConstantInt::getNullValue(getSizeTTy(M))); Builder.CreateCondBr(FnCond, IfThenBB, IfElseBB); @@ -392,30 +421,37 @@ Function *createRegisterGlobalsFunction(Module &M, bool IsHIP) { Builder.CreateBr(IfEndBB); Builder.SetInsertPoint(IfElseBB); - auto *Switch = Builder.CreateSwitch(Flags, IfEndBB); + auto *Switch = Builder.CreateSwitch(Kind, IfEndBB); // Create global variable registration code. Builder.SetInsertPoint(SwGlobalBB); - Builder.CreateCall(RegVar, {RegGlobalsFn->arg_begin(), Addr, Name, Name, - ConstantInt::get(Type::getInt32Ty(C), 0), Size, - ConstantInt::get(Type::getInt32Ty(C), 0), - ConstantInt::get(Type::getInt32Ty(C), 0)}); + Builder.CreateCall(RegVar, + {RegGlobalsFn->arg_begin(), Addr, Name, Name, Extern, Size, + Const, ConstantInt::get(Type::getInt32Ty(C), 0)}); Builder.CreateBr(IfEndBB); - Switch->addCase(Builder.getInt32(OffloadGlobalEntry), SwGlobalBB); + Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalEntry), + SwGlobalBB); // Create managed variable registration code. Builder.SetInsertPoint(SwManagedBB); Builder.CreateBr(IfEndBB); - Switch->addCase(Builder.getInt32(OffloadGlobalManagedEntry), SwManagedBB); + Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalManagedEntry), + SwManagedBB); // Create surface variable registration code. Builder.SetInsertPoint(SwSurfaceBB); + Builder.CreateCall( + RegSurface, {RegGlobalsFn->arg_begin(), Addr, Name, Name, Data, Extern}); Builder.CreateBr(IfEndBB); - Switch->addCase(Builder.getInt32(OffloadGlobalSurfaceEntry), SwSurfaceBB); + Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalSurfaceEntry), + SwSurfaceBB); // Create texture variable registration code. Builder.SetInsertPoint(SwTextureBB); + Builder.CreateCall(RegTexture, {RegGlobalsFn->arg_begin(), Addr, Name, Name, + Data, Normalized, Extern}); Builder.CreateBr(IfEndBB); - Switch->addCase(Builder.getInt32(OffloadGlobalTextureEntry), SwTextureBB); + Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalTextureEntry), + SwTextureBB); Builder.SetInsertPoint(IfEndBB); auto *NewEntry = Builder.CreateInBoundsGEP( diff --git a/clang/tools/libclang/CXIndexDataConsumer.cpp b/clang/tools/libclang/CXIndexDataConsumer.cpp index 5ca484fbc8cd82bf7175489e6f891f4975e60805..c1022263a51280263812244fb19047f08971650f 100644 --- a/clang/tools/libclang/CXIndexDataConsumer.cpp +++ b/clang/tools/libclang/CXIndexDataConsumer.cpp @@ -1074,8 +1074,8 @@ CXIndexDataConsumer::getClientContainerForDC(const DeclContext *DC) const { return DC ? ContainerMap.lookup(DC) : nullptr; } -CXIdxClientFile CXIndexDataConsumer::getIndexFile(const FileEntry *File) { - return File ? FileMap.lookup(File) : nullptr; +CXIdxClientFile CXIndexDataConsumer::getIndexFile(OptionalFileEntryRef File) { + return File ? FileMap.lookup(*File) : nullptr; } CXIdxLoc CXIndexDataConsumer::getIndexLoc(SourceLocation Loc) const { @@ -1104,8 +1104,8 @@ void CXIndexDataConsumer::translateLoc(SourceLocation Loc, if (FID.isInvalid()) return; - - OptionalFileEntryRefDegradesToFileEntryPtr FE = SM.getFileEntryRefForID(FID); + + OptionalFileEntryRef FE = SM.getFileEntryRefForID(FID); if (indexFile) *indexFile = getIndexFile(FE); if (file) diff --git a/clang/tools/libclang/CXIndexDataConsumer.h b/clang/tools/libclang/CXIndexDataConsumer.h index afa2239ed653f9dd19e01afb32ecfd8ef8cf3560..54a3add3a9c8d0b30e9616b4f348297b73f93736 100644 --- a/clang/tools/libclang/CXIndexDataConsumer.h +++ b/clang/tools/libclang/CXIndexDataConsumer.h @@ -460,8 +460,8 @@ private: const DeclContext *getEntityContainer(const Decl *D) const; - CXIdxClientFile getIndexFile(const FileEntry *File); - + CXIdxClientFile getIndexFile(OptionalFileEntryRef File); + CXIdxLoc getIndexLoc(SourceLocation Loc) const; void getEntityInfo(const NamedDecl *D, diff --git a/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp b/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp index f92afd8c3d84a08305033dbd0677bdea3a8c25d9..4c3cb322eacfb3a3b018d5c3fc8a6275255aa278 100644 --- a/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp +++ b/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp @@ -514,8 +514,17 @@ TEST_F(NoreturnDestructorTest, ConditionalOperatorNestedBranchReturns) { class SpecialBoolAnalysis final : public DataflowAnalysis { public: - explicit SpecialBoolAnalysis(ASTContext &Context) - : DataflowAnalysis(Context) {} + explicit SpecialBoolAnalysis(ASTContext &Context, Environment &Env) + : DataflowAnalysis(Context) { + Env.getDataflowAnalysisContext().setSyntheticFieldCallback( + [](QualType Ty) -> llvm::StringMap { + RecordDecl *RD = Ty->getAsRecordDecl(); + if (RD == nullptr || RD->getIdentifier() == nullptr || + RD->getName() != "SpecialBool") + return {}; + return {{"is_set", RD->getASTContext().BoolTy}}; + }); + } static NoopLattice initialElement() { return {}; } @@ -530,67 +539,18 @@ public: if (const auto *E = selectFirst( "call", match(cxxConstructExpr(HasSpecialBoolType).bind("call"), *S, getASTContext()))) { - cast(Env.getValue(*E)) - ->setProperty("is_set", Env.getBoolLiteralValue(false)); + Env.setValue(Env.getResultObjectLocation(*E).getSyntheticField("is_set"), + Env.getBoolLiteralValue(false)); } else if (const auto *E = selectFirst( "call", match(cxxMemberCallExpr(callee(cxxMethodDecl(ofClass( SpecialBoolRecordDecl)))) .bind("call"), *S, getASTContext()))) { - auto &ObjectLoc = - *cast(getImplicitObjectLocation(*E, Env)); - - refreshRecordValue(ObjectLoc, Env) - .setProperty("is_set", Env.getBoolLiteralValue(true)); + if (RecordStorageLocation *ObjectLoc = getImplicitObjectLocation(*E, Env)) + Env.setValue(ObjectLoc->getSyntheticField("is_set"), + Env.getBoolLiteralValue(true)); } } - - ComparisonResult compare(QualType Type, const Value &Val1, - const Environment &Env1, const Value &Val2, - const Environment &Env2) override { - const auto *Decl = Type->getAsCXXRecordDecl(); - if (Decl == nullptr || Decl->getIdentifier() == nullptr || - Decl->getName() != "SpecialBool") - return ComparisonResult::Unknown; - - auto *IsSet1 = cast_or_null(Val1.getProperty("is_set")); - auto *IsSet2 = cast_or_null(Val2.getProperty("is_set")); - if (IsSet1 == nullptr) - return IsSet2 == nullptr ? ComparisonResult::Same - : ComparisonResult::Different; - - if (IsSet2 == nullptr) - return ComparisonResult::Different; - - return Env1.proves(IsSet1->formula()) == Env2.proves(IsSet2->formula()) - ? ComparisonResult::Same - : ComparisonResult::Different; - } - - // Always returns `true` to accept the `MergedVal`. - bool merge(QualType Type, const Value &Val1, const Environment &Env1, - const Value &Val2, const Environment &Env2, Value &MergedVal, - Environment &MergedEnv) override { - const auto *Decl = Type->getAsCXXRecordDecl(); - if (Decl == nullptr || Decl->getIdentifier() == nullptr || - Decl->getName() != "SpecialBool") - return true; - - auto *IsSet1 = cast_or_null(Val1.getProperty("is_set")); - if (IsSet1 == nullptr) - return true; - - auto *IsSet2 = cast_or_null(Val2.getProperty("is_set")); - if (IsSet2 == nullptr) - return true; - - auto &IsSet = MergedEnv.makeAtomicBoolValue(); - MergedVal.setProperty("is_set", IsSet); - if (Env1.proves(IsSet1->formula()) && Env2.proves(IsSet2->formula())) - MergedEnv.assume(IsSet.formula()); - - return true; - } }; class JoinFlowConditionsTest : public Test { @@ -602,7 +562,7 @@ protected: AnalysisInputs( Code, ast_matchers::hasName("target"), [](ASTContext &Context, Environment &Env) { - return SpecialBoolAnalysis(Context); + return SpecialBoolAnalysis(Context, Env); }) .withASTBuildArgs({"-fsyntax-only", "-std=c++17"}), /*VerifyResults=*/[&Match](const llvm::StringMap< @@ -650,7 +610,9 @@ TEST_F(JoinFlowConditionsTest, JoinDistinctButProvablyEquivalentValues) { ASSERT_THAT(FooDecl, NotNull()); auto GetFoo = [FooDecl](const Environment &Env) -> const Formula & { - return cast(Env.getValue(*FooDecl)->getProperty("is_set")) + auto *Loc = + cast(Env.getStorageLocation(*FooDecl)); + return cast(Env.getValue(Loc->getSyntheticField("is_set"))) ->formula(); }; diff --git a/clang/unittests/Basic/FileEntryTest.cpp b/clang/unittests/Basic/FileEntryTest.cpp index dcd196417da731ec9940b9629f23917cbb501d8d..f8a0b4a4edcdaf402ae37aaa273c76a72b25ef19 100644 --- a/clang/unittests/Basic/FileEntryTest.cpp +++ b/clang/unittests/Basic/FileEntryTest.cpp @@ -92,24 +92,6 @@ TEST(FileEntryTest, FileEntryRef) { EXPECT_EQ(CE1, &R1.getFileEntry()); } -TEST(FileEntryTest, OptionalFileEntryRefDegradesToFileEntryPtr) { - FileEntryTestHelper Refs; - OptionalFileEntryRefDegradesToFileEntryPtr M0; - OptionalFileEntryRefDegradesToFileEntryPtr M1 = Refs.addFile("1"); - OptionalFileEntryRefDegradesToFileEntryPtr M2 = Refs.addFile("2"); - OptionalFileEntryRefDegradesToFileEntryPtr M0Also = std::nullopt; - OptionalFileEntryRefDegradesToFileEntryPtr M1Also = - Refs.addFileAlias("1-also", *M1); - - EXPECT_EQ(M0, M0Also); - EXPECT_EQ(StringRef("1"), M1->getName()); - EXPECT_EQ(StringRef("2"), M2->getName()); - EXPECT_EQ(StringRef("1-also"), M1Also->getName()); - - const FileEntry *CE1 = M1; - EXPECT_EQ(CE1, &M1->getFileEntry()); -} - TEST(FileEntryTest, equals) { FileEntryTestHelper Refs; FileEntryRef R1 = Refs.addFile("1"); @@ -126,13 +108,6 @@ TEST(FileEntryTest, equals) { EXPECT_NE(R1, R2); EXPECT_EQ(R1, R1Redirect); EXPECT_EQ(R1, R1Redirect2); - - OptionalFileEntryRefDegradesToFileEntryPtr M1 = R1; - - EXPECT_EQ(M1, &R1.getFileEntry()); - EXPECT_EQ(&R1.getFileEntry(), M1); - EXPECT_NE(M1, &R2.getFileEntry()); - EXPECT_NE(&R2.getFileEntry(), M1); } TEST(FileEntryTest, isSameRef) { diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index 8782cb7c49cad33f9328d6ab64609cec537e6c8a..24b2fd599dc39733911c56a2a7cf64d0ffb8a189 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -13950,6 +13950,19 @@ TEST_F(FormatTest, PullTrivialFunctionDefinitionsIntoSingleLine) { " void f() { int i; } \\\n" " int j;", getLLVMStyleWithColumns(23)); + + verifyFormat( + "void aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n" + " aaaaaaaaaaaaaaaaaa,\n" + " aaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb) {}"); + + constexpr StringRef Code{"void foo() { /* Empty */ }"}; + verifyFormat(Code); + verifyFormat(Code, "void foo() { /* Empty */\n" + "}"); + verifyFormat(Code, "void foo() {\n" + "/* Empty */\n" + "}"); } TEST_F(FormatTest, PullEmptyFunctionDefinitionsIntoSingleLine) { diff --git a/clang/unittests/Format/FormatTestComments.cpp b/clang/unittests/Format/FormatTestComments.cpp index 9770d5090703c777385a22dd2fc76e7d52372eff..c249f4d9333fd079744a0cc22b26af369d641f2c 100644 --- a/clang/unittests/Format/FormatTestComments.cpp +++ b/clang/unittests/Format/FormatTestComments.cpp @@ -386,15 +386,16 @@ TEST_F(FormatTestComments, UnderstandsBlockComments) { " /* Leading comment for bb... */ bbbbbbbbbbbbbbbbbbbbbbbbb);", format("f(aaaaaaaaaaaaaaaaaaaaaaaaa , \n" "/* Leading comment for bb... */ bbbbbbbbbbbbbbbbbbbbbbbbb);")); - EXPECT_EQ( + + verifyFormat( "void aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n" " aaaaaaaaaaaaaaaaaa,\n" - " aaaaaaaaaaaaaaaaaa) { /*aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa*/\n" - "}", - format("void aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n" - " aaaaaaaaaaaaaaaaaa ,\n" - " aaaaaaaaaaaaaaaaaa) { /*aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa*/\n" - "}")); + " aaaaaaaaaaaaaaaaaa) { /*aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa*/ }", + "void aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n" + " aaaaaaaaaaaaaaaaaa ,\n" + " aaaaaaaaaaaaaaaaaa) { /*aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa*/\n" + "}"); + verifyFormat("f(/* aaaaaaaaaaaaaaaaaa = */\n" " aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa);"); diff --git a/clang/www/c_status.html b/clang/www/c_status.html index 91cae138074b335ec9e76e5d8a445fcf923b6b16..47acb1f87b8e3d76448e4918ad87ed5859d22274 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -1151,7 +1151,7 @@ conformance.

Identifier syntax fixes N2939 - Unknown + Clang 15 Remove trigraphs??! diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html index aafed2b6869e4f70c76917ed1dcc72d4594b94db..d09cf616899ad45c1dc6d56ecdebabe77611d0c0 100755 --- a/clang/www/cxx_dr_status.html +++ b/clang/www/cxx_dr_status.html @@ -2511,7 +2511,7 @@ of class templates 412 NAD Can a replacement allocation function be inline? - Yes + Clang 3.4 413 @@ -5117,7 +5117,7 @@ and POD class 873 C++11 Deducing rvalue references in declarative contexts - Yes + Clang 3.0 874 @@ -5519,7 +5519,7 @@ and POD class 952 CD6 Insufficient description of “naming class” - Yes + Clang 2.8 953 @@ -6473,7 +6473,7 @@ and POD class 1111 C++11 Remove dual-scope lookup of member template names - Yes + Clang 3.2 1112 @@ -7169,7 +7169,7 @@ and POD class 1227 CD3 Mixing immediate and non-immediate contexts in deduction failure - Yes + Clang 3.0 1228 @@ -7637,7 +7637,7 @@ and POD class 1305 CD3 alignof applied to array of unknown size - Yes + Clang 3.0 1306 @@ -7889,7 +7889,7 @@ and POD class 1347 CD3 Consistency of auto in multiple-declarator declarations - Yes + Clang 3.1 1348 @@ -7955,7 +7955,7 @@ and POD class 1358 CD3 Unintentionally ill-formed constexpr function template instances - Yes + Clang 3.1 1359 @@ -8681,7 +8681,7 @@ and POD class 1479 CD3 Literal operators and default arguments - Yes + Clang 3.1 1480 @@ -8699,7 +8699,7 @@ and POD class 1482 CD3 Point of declaration of enumeration - Yes + Clang 3.0 1483 @@ -9107,7 +9107,7 @@ and POD class 1550 CD3 Parenthesized throw-expression operand of conditional-expression - Yes + Clang 3.4 1551 @@ -9635,7 +9635,7 @@ and POD class 1638 CD4 Declaring an explicit specialization of a scoped enumeration - Yes + Clang 3.1 1639 @@ -10733,7 +10733,7 @@ and POD class 1821 CD6 Qualified redeclarations in a class member-specification - Yes + Clang 2.9 1822 @@ -11261,7 +11261,7 @@ and POD class 1909 CD4 Member class template with the same name as the class - Yes + Clang 3.7 1910 @@ -11447,7 +11447,7 @@ and POD class 1940 CD4 static_assert in anonymous unions - Yes + Clang 3.5 1941 @@ -11489,13 +11489,13 @@ and POD class 1947 NAD Digit separators following non-octal prefix - Yes + Clang 3.5 1948 NAD exception-specification of replacement global new - Yes + Clang 3.5 1949 @@ -14147,7 +14147,7 @@ and POD class 2390 CD5 Is the argument of __has_cpp_attribute macro-expanded? - Yes + Clang 14 2391 @@ -14243,7 +14243,7 @@ and POD class 2406 CD5 [[fallthrough]] attribute and iteration statements - Yes + Clang 5 2407 @@ -14903,7 +14903,7 @@ and POD class 2516 C++23 Locus of enum-specifier or opaque-enum-declaration - Yes + Clang 3.0 2517 @@ -15533,7 +15533,7 @@ and POD class 2621 C++23 Kind of lookup for using enum declarations - Yes + Clang 16 2622 @@ -15671,7 +15671,7 @@ and POD class 2644 C++23 Incorrect comment in example - Yes + Clang 8 2645 @@ -15707,7 +15707,7 @@ and POD class 2650 C++23 Incorrect example for ill-formed non-type template arguments - Yes + Clang 17 2651 diff --git a/compiler-rt/lib/asan/asan_mac.cpp b/compiler-rt/lib/asan/asan_mac.cpp index 5d5146e0cde0ba9b880aacc1410d47bc0058f8c5..1b0e9b3fe006049deff5c9a91b34d07185c63be1 100644 --- a/compiler-rt/lib/asan/asan_mac.cpp +++ b/compiler-rt/lib/asan/asan_mac.cpp @@ -139,9 +139,11 @@ typedef void (*dispatch_mach_handler_function_t)(void *context, dispatch_mach_reason reason, dispatch_mach_msg_t message, mach_error_t error); +# if !defined(MISSING_BLOCKS_SUPPORT) typedef void (^dispatch_mach_handler_t)(dispatch_mach_reason reason, dispatch_mach_msg_t message, mach_error_t error); +# endif // A wrapper for the ObjC blocks used to support libdispatch. typedef struct { diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt index 069d33bfd3d309bc9b096bcd5a288ed3a1584836..ea72c595a9b807c9a396bf6d9ddd07f7e5948a4c 100644 --- a/compiler-rt/lib/builtins/CMakeLists.txt +++ b/compiler-rt/lib/builtins/CMakeLists.txt @@ -38,6 +38,13 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) include(UseLibtool) endif() include(AddCompilerRT) + + if(MINGW) + # Simplified version of what's set in cmake/config-ix.cmake; not including + # builtins, which are linked separately. + set(MINGW_LIBRARIES mingw32 moldname mingwex msvcrt advapi32 shell32 + user32 kernel32 mingw32 moldname mingwex msvcrt) + endif() endif() if (COMPILER_RT_STANDALONE_BUILD) @@ -881,12 +888,14 @@ if(COMPILER_RT_BUILD_STANDALONE_LIBATOMIC) endif() foreach (arch ${BUILTIN_SUPPORTED_ARCH}) if(CAN_TARGET_${arch}) + list(APPEND COMPILER_RT_LIBATOMIC_LINK_LIBS_${arch} clang_rt.builtins-${arch}) + append_list_if(MINGW "${MINGW_LIBRARIES}" COMPILER_RT_LIBATOMIC_LINK_LIBS_${arch}) add_compiler_rt_runtime(clang_rt.atomic ${BUILTIN_TYPE} ARCHS ${arch} SOURCES atomic.c LINK_FLAGS ${COMPILER_RT_LIBATOMIC_LINK_FLAGS} - LINK_LIBS clang_rt.builtins-${arch} + LINK_LIBS ${COMPILER_RT_LIBATOMIC_LINK_LIBS_${arch}} PARENT_TARGET builtins-standalone-atomic) endif() endforeach() diff --git a/compiler-rt/lib/hwasan/hwasan_flags.inc b/compiler-rt/lib/hwasan/hwasan_flags.inc index 978fa46b705cb9e6686fe1a0f48dd03aa5be6ae3..058a0457b9e7f65d3a39103a18eab6e948a28644 100644 --- a/compiler-rt/lib/hwasan/hwasan_flags.inc +++ b/compiler-rt/lib/hwasan/hwasan_flags.inc @@ -84,3 +84,10 @@ HWASAN_FLAG(bool, malloc_bisect_dump, false, // are untagged before the call. HWASAN_FLAG(bool, fail_without_syscall_abi, true, "Exit if fail to request relaxed syscall ABI.") + +HWASAN_FLAG( + uptr, fixed_shadow_base, -1, + "If not -1, HWASan will attempt to allocate the shadow at this address, " + "instead of choosing one dynamically." + "Tip: this can be combined with the compiler option, " + "-hwasan-mapping-offset, to optimize the instrumentation.") diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cpp b/compiler-rt/lib/hwasan/hwasan_linux.cpp index 81226da976d1161ad7f614413e5387cfec9ba51c..f01fa427641347b8a44de7da8260ecaf3b12a80e 100644 --- a/compiler-rt/lib/hwasan/hwasan_linux.cpp +++ b/compiler-rt/lib/hwasan/hwasan_linux.cpp @@ -106,8 +106,12 @@ static uptr GetHighMemEnd() { } static void InitializeShadowBaseAddress(uptr shadow_size_bytes) { - __hwasan_shadow_memory_dynamic_address = - FindDynamicShadowStart(shadow_size_bytes); + if (flags()->fixed_shadow_base != (uptr)-1) { + __hwasan_shadow_memory_dynamic_address = flags()->fixed_shadow_base; + } else { + __hwasan_shadow_memory_dynamic_address = + FindDynamicShadowStart(shadow_size_bytes); + } } static void MaybeDieIfNoTaggingAbi(const char *message) { diff --git a/compiler-rt/lib/interception/interception_win.cpp b/compiler-rt/lib/interception/interception_win.cpp index f3e62419beff1284748008bd6df4dff773ee5953..f1a549b938e202d359f7c915baac5042b0f01726 100644 --- a/compiler-rt/lib/interception/interception_win.cpp +++ b/compiler-rt/lib/interception/interception_win.cpp @@ -578,6 +578,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0x018a: // mov al, byte ptr [rcx] return 2; + case 0x058A: // 8A 05 XX XX XX XX : mov al, byte ptr [XX XX XX XX] case 0x058B: // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX] if (rel_offset) *rel_offset = 2; diff --git a/compiler-rt/lib/orc/macho_platform.cpp b/compiler-rt/lib/orc/macho_platform.cpp index 73b17a0799c4c60332f27b38d38ac5d6d2ed66ea..e3a1cdf3c4fcc334e13f34cb27b41362a0e96336 100644 --- a/compiler-rt/lib/orc/macho_platform.cpp +++ b/compiler-rt/lib/orc/macho_platform.cpp @@ -352,36 +352,13 @@ private: Error requestPushSymbols(JITDylibState &JDS, span> Symbols); - /// Visits the symbol table for the JITDylib associated with DSOHandle. - /// Visitor should be callable as - /// - /// void (size_t, - /// std::optional>) - /// - /// The visitor function will be called for each element of the Symbols, but - /// in an arbitrary order. The first argument of the callback will indicate - /// the index of the result. The second argument will be std::nullopt (if the - /// symbol at the given index was not present in the symbol table), or a - /// pair containing the symbol's address and flags. - /// - /// This function will remove all elements of Symbols that are found, leaving - /// only the symbols that were not. This allows it to dovetail with - /// requestPushSymbols, enabling the following idiom: - /// - /// ... - /// visitSymbolAddrs(DSO, Symbols); - /// if (!Symbols.empty()) { - /// requestPushSymbols(DSO, Symbols); - /// visitSymbolAddrs(DSO, Symbols); - /// for (auto &Sym : Symbols) { - /// -- handle symbols that were not found -- - /// } - /// } - /// - template - void visitSymbolAddrs(JITDylibState &JDS, - std::vector> &Symbols, - VisitorFn &&Visit); + /// Attempts to look up the given symbols locally, requesting a push from the + /// remote if they're not found. Results are written to the Result span, which + /// must have the same size as the Symbols span. + Error + lookupSymbols(JITDylibState &JDS, std::unique_lock &JDStatesLock, + span> Result, + span> Symbols); bool lookupUnwindSections(void *Addr, unw_dynamic_unwind_sections &Info); @@ -830,7 +807,7 @@ int MachOPlatformRuntimeState::dlclose(void *DSOHandle) { } void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, const char *Symbol) { - std::lock_guard Lock(JDStatesMutex); + std::unique_lock Lock(JDStatesMutex); auto *JDS = getJITDylibStateByHeader(DSOHandle); if (!JDS) { std::ostringstream ErrStream; @@ -839,41 +816,16 @@ void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, const char *Symbol) { return nullptr; } - std::string MangledName("_"); - MangledName += Symbol; - std::vector> Symbols; - Symbols.push_back({MangledName, false}); - - ExecutorAddr Result; - using ElemResult = - std::optional>; - - // Try to resolve the symbol in the local symbol tables. - visitSymbolAddrs(*JDS, Symbols, [&](size_t Idx, ElemResult E) { - if (E) - Result = E->first; - }); + std::string MangledName = std::string("_") + Symbol; + std::pair Lookup(MangledName, false); + std::pair Result; - // Return early if we found it. - if (Symbols.empty()) - return Result.toPtr(); - - // Otherwise call back to the controller to try to request that the symbol - // be materialized. - if (auto Err = requestPushSymbols(*JDS, {Symbols.data(), Symbols.size()})) { + if (auto Err = lookupSymbols(*JDS, Lock, {&Result, 1}, {&Lookup, 1})) { DLFcnError = toString(std::move(Err)); return nullptr; } - // Try another local resolution. - visitSymbolAddrs(*JDS, Symbols, [&](size_t Idx, ElemResult E) { - if (E) - Result = E->first; - }); - - // At this point Result has either been set (if we found the symbol) or is - // still null (if we didn't). Either way it's the right value. - return Result.toPtr(); + return Result.first.toPtr(); } int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg, @@ -965,22 +917,71 @@ Error MachOPlatformRuntimeState::requestPushSymbols( return OpErr; } -template -void MachOPlatformRuntimeState::visitSymbolAddrs( - JITDylibState &JDS, std::vector> &Symbols, - VisitorFn &&Visit) { - - std::vector> RemainingSymbols; - +Error MachOPlatformRuntimeState::lookupSymbols( + JITDylibState &JDS, std::unique_lock &JDStatesLock, + span> Result, + span> Symbols) { + assert(JDStatesLock.owns_lock() && + "JDStatesLock should be locked at call-site"); + assert(Result.size() == Symbols.size() && + "Results and Symbols span sizes should match"); + + // Make an initial pass over the local symbol table. + std::vector MissingSymbolIndexes; for (size_t Idx = 0; Idx != Symbols.size(); ++Idx) { auto I = JDS.SymbolTable.find(Symbols[Idx].first); if (I != JDS.SymbolTable.end()) - Visit(Idx, I->second); + Result[Idx] = I->second; else - RemainingSymbols.push_back(Symbols[Idx]); + MissingSymbolIndexes.push_back(Idx); } - Symbols = std::move(RemainingSymbols); + // If everything has been resolved already then bail out early. + if (MissingSymbolIndexes.empty()) + return Error::success(); + + // Otherwise call back to the controller to try to request that the symbol + // be materialized. + std::vector> MissingSymbols; + MissingSymbols.reserve(MissingSymbolIndexes.size()); + ORC_RT_DEBUG({ + printdbg("requesting push of %i missing symbols...\n", + MissingSymbolIndexes.size()); + }); + for (auto MissingIdx : MissingSymbolIndexes) + MissingSymbols.push_back(Symbols[MissingIdx]); + + JDStatesLock.unlock(); + if (auto Err = requestPushSymbols( + JDS, {MissingSymbols.data(), MissingSymbols.size()})) + return Err; + JDStatesLock.lock(); + + // Try to resolve the previously missing symbols locally. + std::vector MissingRequiredSymbols; + for (auto MissingIdx : MissingSymbolIndexes) { + auto I = JDS.SymbolTable.find(Symbols[MissingIdx].first); + if (I != JDS.SymbolTable.end()) + Result[MissingIdx] = I->second; + else { + if (Symbols[MissingIdx].second) + MissingRequiredSymbols.push_back(MissingIdx); + else + Result[MissingIdx] = {ExecutorAddr(), {}}; + } + } + + // Error out if any missing symbols could not be resolved. + if (!MissingRequiredSymbols.empty()) { + std::ostringstream ErrStream; + ErrStream << "Lookup could not find required symbols: [ "; + for (auto MissingIdx : MissingRequiredSymbols) + ErrStream << "\"" << Symbols[MissingIdx].first << "\" "; + ErrStream << "]"; + return make_error(ErrStream.str()); + } + + return Error::success(); } // eh-frame registration functions. diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index c5b0b34f2d8af03dfb2bf90ee252a4fba8050752..137115996748ce39f82e3d9597b08098b8de50a8 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -54,6 +54,12 @@ int __llvm_profile_is_continuous_mode_enabled(void); */ void __llvm_profile_enable_continuous_mode(void); +/*! + * \brief Disable continuous mode. + * + */ +void __llvm_profile_disable_continuous_mode(void); + /*! * \brief Set the page size. * diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index cd1f067bd188e47d6cc031037bf66e6bf60df3c0..af52804b2b532cc3a8002016f998ad9cf390b3af 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -33,6 +33,10 @@ COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) { ContinuouslySyncProfile = 1; } +COMPILER_RT_VISIBILITY void __llvm_profile_disable_continuous_mode(void) { + ContinuouslySyncProfile = 0; +} + COMPILER_RT_VISIBILITY void __llvm_profile_set_page_size(unsigned PS) { PageSize = PS; } diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c index 1685b30b9492a64bf02865c5bfa56f937370c3a0..745c567f21673b2028598cffe06d52b389fa70c5 100644 --- a/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/compiler-rt/lib/profile/InstrProfilingFile.c @@ -806,6 +806,7 @@ static int parseFilenamePattern(const char *FilenamePat, if (__llvm_profile_is_continuous_mode_enabled()) { PROF_WARN("%%c specifier can only be specified once in %s.\n", FilenamePat); + __llvm_profile_disable_continuous_mode(); return -1; } #if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32) diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h index 25ad11dbf7ee549f159e709bf85f037b6e0655d3..65ddc488370a7d28b8939bb3301aa286c3491b0e 100644 --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -14,6 +14,7 @@ #include "flags.h" #include "flags_parser.h" #include "local_cache.h" +#include "mem_map.h" #include "memtag.h" #include "options.h" #include "quarantine.h" @@ -935,8 +936,7 @@ public: uptr getRingBufferSize() { initThreadMaybe(); - auto *RingBuffer = getRingBuffer(); - return RingBuffer ? ringBufferSizeInBytes(RingBuffer->Size) : 0; + return RingBufferElements ? ringBufferSizeInBytes(RingBufferElements) : 0; } static bool setRingBufferSizeForBuffer(char *Buffer, size_t Size) { @@ -966,8 +966,9 @@ public: static void getErrorInfo(struct scudo_error_info *ErrorInfo, uintptr_t FaultAddr, const char *DepotPtr, const char *RegionInfoPtr, const char *RingBufferPtr, - const char *Memory, const char *MemoryTags, - uintptr_t MemoryAddr, size_t MemorySize) { + size_t RingBufferSize, const char *Memory, + const char *MemoryTags, uintptr_t MemoryAddr, + size_t MemorySize) { *ErrorInfo = {}; if (!allocatorSupportsMemoryTagging() || MemoryAddr + MemorySize < MemoryAddr) @@ -986,7 +987,7 @@ public: // Check the ring buffer. For primary allocations this will only find UAF; // for secondary allocations we can find either UAF or OOB. getRingBufferErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot, - RingBufferPtr); + RingBufferPtr, RingBufferSize); // Check for OOB in the 28 blocks surrounding the 3 we checked earlier. // Beyond that we are likely to hit false positives. @@ -1051,15 +1052,15 @@ private: atomic_u32 DeallocationTid; }; - MemMapT MemMap; atomic_uptr Pos; - u32 Size; // An array of Size (at least one) elements of type Entry is immediately // following to this struct. }; // Pointer to memory mapped area starting with AllocationRingBuffer struct, // and immediately followed by Size elements of type Entry. char *RawRingBuffer = {}; + u32 RingBufferElements = 0; + MemMapT RawRingBufferMap; // The following might get optimized out by the compiler. NOINLINE void performSanityChecks() { @@ -1267,7 +1268,7 @@ private: u32 DeallocationTid) { uptr Pos = atomic_fetch_add(&getRingBuffer()->Pos, 1, memory_order_relaxed); typename AllocationRingBuffer::Entry *Entry = - getRingBufferEntry(RawRingBuffer, Pos % getRingBuffer()->Size); + getRingBufferEntry(RawRingBuffer, Pos % RingBufferElements); // First invalidate our entry so that we don't attempt to interpret a // partially written state in getSecondaryErrorInfo(). The fences below @@ -1408,17 +1409,19 @@ private: size_t &NextErrorReport, uintptr_t FaultAddr, const StackDepot *Depot, - const char *RingBufferPtr) { + const char *RingBufferPtr, + size_t RingBufferSize) { auto *RingBuffer = reinterpret_cast(RingBufferPtr); - if (!RingBuffer || RingBuffer->Size == 0) + size_t RingBufferElements = ringBufferElementsFromBytes(RingBufferSize); + if (!RingBuffer || RingBufferElements == 0) return; uptr Pos = atomic_load_relaxed(&RingBuffer->Pos); - for (uptr I = Pos - 1; - I != Pos - 1 - RingBuffer->Size && NextErrorReport != NumErrorReports; + for (uptr I = Pos - 1; I != Pos - 1 - RingBufferElements && + NextErrorReport != NumErrorReports; --I) { - auto *Entry = getRingBufferEntry(RingBufferPtr, I % RingBuffer->Size); + auto *Entry = getRingBufferEntry(RingBufferPtr, I % RingBufferElements); uptr EntryPtr = atomic_load_relaxed(&Entry->Ptr); if (!EntryPtr) continue; @@ -1508,9 +1511,8 @@ private: getPageSizeCached()), "scudo:ring_buffer"); RawRingBuffer = reinterpret_cast(MemMap.getBase()); - auto *RingBuffer = reinterpret_cast(RawRingBuffer); - RingBuffer->MemMap = MemMap; - RingBuffer->Size = AllocationRingBufferSize; + RawRingBufferMap = MemMap; + RingBufferElements = AllocationRingBufferSize; static_assert(sizeof(AllocationRingBuffer) % alignof(typename AllocationRingBuffer::Entry) == 0, @@ -1520,16 +1522,23 @@ private: void unmapRingBuffer() { auto *RingBuffer = getRingBuffer(); if (RingBuffer != nullptr) { - MemMapT MemMap = RingBuffer->MemMap; - MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); + RawRingBufferMap.unmap(RawRingBufferMap.getBase(), + RawRingBufferMap.getCapacity()); } RawRingBuffer = nullptr; } - static constexpr size_t ringBufferSizeInBytes(u32 AllocationRingBufferSize) { + static constexpr size_t ringBufferSizeInBytes(u32 RingBufferElements) { return sizeof(AllocationRingBuffer) + - AllocationRingBufferSize * - sizeof(typename AllocationRingBuffer::Entry); + RingBufferElements * sizeof(typename AllocationRingBuffer::Entry); + } + + static constexpr size_t ringBufferElementsFromBytes(size_t Bytes) { + if (Bytes < sizeof(AllocationRingBuffer)) { + return 0; + } + return (Bytes - sizeof(AllocationRingBuffer)) / + sizeof(typename AllocationRingBuffer::Entry); } inline AllocationRingBuffer *getRingBuffer() { diff --git a/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp b/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp index 74456450a476126536c96e20090533c898e05190..5b01ebe11c095239ef8fae4e921a45f11ec9f8c6 100644 --- a/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp +++ b/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp @@ -46,14 +46,11 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t *Data, size_t Size) { } std::string RingBufferBytes = FDP.ConsumeRemainingBytesAsString(); - // RingBuffer is too short. - if (!AllocatorT::setRingBufferSizeForBuffer(RingBufferBytes.data(), - RingBufferBytes.size())) - return 0; scudo_error_info ErrorInfo; AllocatorT::getErrorInfo(&ErrorInfo, FaultAddr, StackDepot.data(), - RegionInfo.data(), RingBufferBytes.data(), Memory, - MemoryTags, MemoryAddr, MemorySize); + RegionInfo.data(), RingBufferBytes.data(), + RingBufferBytes.size(), Memory, MemoryTags, + MemoryAddr, MemorySize); return 0; } diff --git a/compiler-rt/lib/scudo/standalone/include/scudo/interface.h b/compiler-rt/lib/scudo/standalone/include/scudo/interface.h index a664b9825f2091e88f37f3fc6872abb3a83dbd3a..a2dedea910cc08c44feb3db485d5cafee9b81348 100644 --- a/compiler-rt/lib/scudo/standalone/include/scudo/interface.h +++ b/compiler-rt/lib/scudo/standalone/include/scudo/interface.h @@ -20,6 +20,19 @@ __attribute__((weak)) const char *__scudo_default_options(void); __attribute__((weak)) void __scudo_allocate_hook(void *ptr, size_t size); __attribute__((weak)) void __scudo_deallocate_hook(void *ptr); +// `realloc` involves both deallocation and allocation but they are not reported +// atomically. In one specific case which may keep taking a snapshot right in +// the middle of `realloc` reporting the deallocation and allocation, it may +// confuse the user by missing memory from `realloc`. To alleviate that case, +// define the two `realloc` hooks to get the knowledge of the bundled hook +// calls. These hooks are optional and should only be used when a hooks user +// wants to track reallocs more closely. +// +// See more details in the comment of `realloc` in wrapper_c.inc. +__attribute__((weak)) void +__scudo_realloc_allocate_hook(void *old_ptr, void *new_ptr, size_t size); +__attribute__((weak)) void __scudo_realloc_deallocate_hook(void *old_ptr); + void __scudo_print_stats(void); typedef void (*iterate_callback)(uintptr_t base, size_t size, void *arg); diff --git a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp index 150688b5b70a54ff0ea57b3a22ad66923fc5fa7d..f5e17d7214863c8b828ca44d2eeb880c91259f8b 100644 --- a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp @@ -61,8 +61,14 @@ struct AllocContext { struct DeallocContext { void *Ptr; }; +struct ReallocContext { + void *AllocPtr; + void *DeallocPtr; + size_t Size; +}; static AllocContext AC; static DeallocContext DC; +static ReallocContext RC; #if (SCUDO_ENABLE_HOOKS_TESTS == 1) __attribute__((visibility("default"))) void __scudo_allocate_hook(void *Ptr, @@ -73,6 +79,28 @@ __attribute__((visibility("default"))) void __scudo_allocate_hook(void *Ptr, __attribute__((visibility("default"))) void __scudo_deallocate_hook(void *Ptr) { DC.Ptr = Ptr; } +__attribute__((visibility("default"))) void +__scudo_realloc_allocate_hook(void *OldPtr, void *NewPtr, size_t Size) { + // Verify that __scudo_realloc_deallocate_hook is called first and set the + // right pointer. + EXPECT_EQ(OldPtr, RC.DeallocPtr); + RC.AllocPtr = NewPtr; + RC.Size = Size; + + // Note that this is only used for testing. In general, only one pair of hooks + // will be invoked in `realloc`. if __scudo_realloc_*_hook are not defined, + // it'll call the general hooks only. To make the test easier, we call the + // general one here so that either case (whether __scudo_realloc_*_hook are + // defined) will be verified without separating them into different tests. + __scudo_allocate_hook(NewPtr, Size); +} +__attribute__((visibility("default"))) void +__scudo_realloc_deallocate_hook(void *Ptr) { + RC.DeallocPtr = Ptr; + + // See the comment in the __scudo_realloc_allocate_hook above. + __scudo_deallocate_hook(Ptr); +} #endif // (SCUDO_ENABLE_HOOKS_TESTS == 1) } @@ -88,6 +116,7 @@ protected: void *InvalidPtr = reinterpret_cast(0xdeadbeef); AC.Ptr = InvalidPtr; DC.Ptr = InvalidPtr; + RC.AllocPtr = RC.DeallocPtr = InvalidPtr; } } void verifyAllocHookPtr(UNUSED void *Ptr) { @@ -102,6 +131,13 @@ protected: if (SCUDO_ENABLE_HOOKS_TESTS) EXPECT_EQ(Ptr, DC.Ptr); } + void verifyReallocHookPtrs(UNUSED void *OldPtr, void *NewPtr, size_t Size) { + if (SCUDO_ENABLE_HOOKS_TESTS) { + EXPECT_EQ(OldPtr, RC.DeallocPtr); + EXPECT_EQ(NewPtr, RC.AllocPtr); + EXPECT_EQ(Size, RC.Size); + } + } }; using ScudoWrappersCDeathTest = ScudoWrappersCTest; @@ -297,6 +333,7 @@ TEST_F(ScudoWrappersCDeathTest, Realloc) { verifyAllocHookSize(Size * 2U); verifyDeallocHookPtr(OldP); } + verifyReallocHookPtrs(OldP, P, Size * 2U); invalidateHookPtrs(); OldP = P; @@ -312,6 +349,7 @@ TEST_F(ScudoWrappersCDeathTest, Realloc) { verifyAllocHookPtr(P); verifyAllocHookSize(Size / 2U); } + verifyReallocHookPtrs(OldP, P, Size / 2U); free(P); EXPECT_DEATH(P = realloc(P, Size), ""); diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c.inc b/compiler-rt/lib/scudo/standalone/wrappers_c.inc index 0413ea49eac0893d23350ceb6f824af9ced1c051..56d8ef20156e2877ea81255972d43d0a6a1421c7 100644 --- a/compiler-rt/lib/scudo/standalone/wrappers_c.inc +++ b/compiler-rt/lib/scudo/standalone/wrappers_c.inc @@ -27,6 +27,24 @@ static void reportDeallocation(void *ptr) { if (__scudo_deallocate_hook) __scudo_deallocate_hook(ptr); } +static void reportReallocAllocation(void *old_ptr, void *new_ptr, size_t size) { + DCHECK_NE(new_ptr, nullptr); + + if (SCUDO_ENABLE_HOOKS) { + if (__scudo_realloc_allocate_hook) + __scudo_realloc_allocate_hook(old_ptr, new_ptr, size); + else if (__scudo_allocate_hook) + __scudo_allocate_hook(new_ptr, size); + } +} +static void reportReallocDeallocation(void *old_ptr) { + if (SCUDO_ENABLE_HOOKS) { + if (__scudo_realloc_deallocate_hook) + __scudo_realloc_deallocate_hook(old_ptr); + else if (__scudo_deallocate_hook) + __scudo_deallocate_hook(old_ptr); + } +} extern "C" { @@ -183,16 +201,17 @@ INTERFACE WEAK void *SCUDO_PREFIX(realloc)(void *ptr, size_t size) { // new pointer. Before the reporting of both operations has been done, another // thread may get the old pointer from `malloc`. It may be misinterpreted as // double-use if it's not handled properly on the hook side. - reportDeallocation(ptr); + reportReallocDeallocation(ptr); void *NewPtr = SCUDO_ALLOCATOR.reallocate(ptr, size, SCUDO_MALLOC_ALIGNMENT); if (NewPtr != nullptr) { // Note that even if NewPtr == ptr, the size has changed. We still need to // report the new size. - reportAllocation(NewPtr, size); + reportReallocAllocation(/*OldPtr=*/ptr, NewPtr, size); } else { // If `realloc` fails, the old pointer is not released. Report the old - // pointer as allocated back. - reportAllocation(ptr, SCUDO_ALLOCATOR.getAllocSize(ptr)); + // pointer as allocated again. + reportReallocAllocation(/*OldPtr=*/ptr, /*NewPtr=*/ptr, + SCUDO_ALLOCATOR.getAllocSize(ptr)); } return scudo::setErrnoOnNull(NewPtr); diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp b/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp index f203615ab3602d4e7ef755fb148fc2b99348cf9c..21694c3f17fe5788bedaecd546dff96d70b6bcdd 100644 --- a/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp +++ b/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp @@ -44,10 +44,9 @@ INTERFACE void __scudo_get_error_info( const char *ring_buffer, size_t ring_buffer_size, const char *memory, const char *memory_tags, uintptr_t memory_addr, size_t memory_size) { (void)(stack_depot_size); - (void)(ring_buffer_size); Allocator.getErrorInfo(error_info, fault_addr, stack_depot, region_info, - ring_buffer, memory, memory_tags, memory_addr, - memory_size); + ring_buffer, ring_buffer_size, memory, memory_tags, + memory_addr, memory_size); } INTERFACE const char *__scudo_get_stack_depot_addr() { diff --git a/compiler-rt/test/hwasan/TestCases/Linux/fixed-shadow.c b/compiler-rt/test/hwasan/TestCases/Linux/fixed-shadow.c new file mode 100644 index 0000000000000000000000000000000000000000..4ff1d3e64c1d0e82bd39d0bc3478fbdc9233204f --- /dev/null +++ b/compiler-rt/test/hwasan/TestCases/Linux/fixed-shadow.c @@ -0,0 +1,76 @@ +// Test fixed shadow base functionality. +// +// Default compiler instrumentation works with any shadow base (dynamic or fixed). +// RUN: %clang_hwasan %s -o %t && %run %t +// RUN: %clang_hwasan %s -o %t && HWASAN_OPTIONS=fixed_shadow_base=263878495698944 %run %t +// RUN: %clang_hwasan %s -o %t && HWASAN_OPTIONS=fixed_shadow_base=4398046511104 %run %t +// +// If -hwasan-mapping-offset is set, then the fixed_shadow_base needs to match. +// RUN: %clang_hwasan %s -mllvm -hwasan-mapping-offset=263878495698944 -o %t && HWASAN_OPTIONS=fixed_shadow_base=263878495698944 %run %t +// RUN: %clang_hwasan %s -mllvm -hwasan-mapping-offset=4398046511104 -o %t && HWASAN_OPTIONS=fixed_shadow_base=4398046511104 %run %t +// RUN: %clang_hwasan %s -mllvm -hwasan-mapping-offset=263878495698944 -o %t && HWASAN_OPTIONS=fixed_shadow_base=4398046511104 not %run %t +// RUN: %clang_hwasan %s -mllvm -hwasan-mapping-offset=4398046511104 -o %t && HWASAN_OPTIONS=fixed_shadow_base=263878495698944 not %run %t +// +// Note: if fixed_shadow_base is not set, compiler-rt will dynamically choose a +// shadow base, which has a tiny but non-zero probability of matching the +// compiler instrumentation. To avoid test flake, we do not test this case. +// +// Assume 48-bit VMA +// REQUIRES: aarch64-target-arch +// +// REQUIRES: Clang +// +// UNSUPPORTED: android + +#include +#include +#include +#include +#include +#include + +int main() { + __hwasan_enable_allocator_tagging(); + + // We test that the compiler instrumentation is able to access shadow memory + // for many different addresses. If we only test a small number of addresses, + // it might work by chance even if the shadow base does not match between the + // compiler instrumentation and compiler-rt. + void **mmaps[256]; + // 48-bit VMA + for (int i = 0; i < 256; i++) { + unsigned long long addr = (i * (1ULL << 40)); + + void *p = mmap((void *)addr, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + // We don't use MAP_FIXED, to avoid overwriting critical memory. + // However, if we don't get allocated the requested address, it + // isn't a useful test. + if ((unsigned long long)p != addr) { + munmap(p, 4096); + mmaps[i] = MAP_FAILED; + } else { + mmaps[i] = p; + } + } + + int failures = 0; + for (int i = 0; i < 256; i++) { + if (mmaps[i] == MAP_FAILED) { + failures++; + } else { + printf("%d %p\n", i, mmaps[i]); + munmap(mmaps[i], 4096); + } + } + + // We expect roughly 17 failures: + // - the page at address zero + // - 16 failures because the shadow memory takes up 1/16th of the address space + // We could also get unlucky e.g., if libraries or binaries are loaded into the + // exact addresses where we tried to map. + // To avoid test flake, we allow some margin of error. + printf("Failed: %d\n", failures); + assert(failures < 48); + return 0; +} diff --git a/compiler-rt/test/orc/TestCases/Darwin/arm64/Inputs/ret_self.S b/compiler-rt/test/orc/TestCases/Darwin/arm64/Inputs/ret_self.S new file mode 100644 index 0000000000000000000000000000000000000000..fce02b2182ab67064de2d818173389ae3091e2d8 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Darwin/arm64/Inputs/ret_self.S @@ -0,0 +1,11 @@ + .section __TEXT,__text,regular,pure_instructions + .build_version macos, 14, 0 sdk_version 14, 4 + .globl _ret_self + .p2align 2 +_ret_self: + adrp x0, _ret_self@PAGE + add x0, x0, _ret_self@PAGEOFF + ret + +.subsections_via_symbols + diff --git a/compiler-rt/test/orc/TestCases/Darwin/arm64/trivial-dlsym.c b/compiler-rt/test/orc/TestCases/Darwin/arm64/trivial-dlsym.c new file mode 100644 index 0000000000000000000000000000000000000000..b597f5dc2fe68a4c90b37dc18263124376e3b865 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Darwin/arm64/trivial-dlsym.c @@ -0,0 +1,44 @@ +// Test that __orc_rt_macho_jit_dlsym works as expected. +// +// RUN: %clang -c -o %t.sym.o %p/Inputs/ret_self.S +// RUN: %clang -c -o %t.test.o %s +// RUN: %llvm_jitlink \ +// RUN: -alias Platform:_dlopen=___orc_rt_macho_jit_dlopen \ +// RUN: -alias Platform:_dlsym=___orc_rt_macho_jit_dlsym \ +// RUN: -alias Platform:_dlclose=___orc_rt_macho_jit_dlclose \ +// RUN: %t.test.o -lextra_sym -jd extra_sym %t.sym.o | FileCheck %s + +// CHECK: entering main +// CHECK-NEXT: found "ret_self" at +// CHECK-NEXT: address of "ret_self" is consistent +// CHECK-NEXT: leaving main + +int printf(const char *restrict format, ...); +void *dlopen(const char *path, int mode); +void *dlsym(void *handle, const char *symbol); +int dlclose(void *handle); + +int main(int argc, char *argv[]) { + printf("entering main\n"); + void *H = dlopen("extra_sym", 0); + if (!H) { + printf("failed\n"); + return -1; + } + + void *(*ret_self)(void) = (void *(*)(void))dlsym(H, "ret_self"); + if (ret_self) + printf("found \"ret_self\" at %p\n", ret_self); + else + printf("failed to find \"ret_self\" via dlsym\n"); + + printf("address of \"ret_self\" is %s\n", + ret_self() == ret_self ? "consistent" : "inconsistent"); + + if (dlclose(H) == -1) { + printf("failed\n"); + return -1; + } + printf("leaving main\n"); + return 0; +} diff --git a/compiler-rt/test/orc/TestCases/Darwin/x86-64/Inputs/ret_self.S b/compiler-rt/test/orc/TestCases/Darwin/x86-64/Inputs/ret_self.S new file mode 100644 index 0000000000000000000000000000000000000000..bcea9868f18d3b81537437ad5f133cb842bfcd6d --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Darwin/x86-64/Inputs/ret_self.S @@ -0,0 +1,12 @@ +// A function that returns its own address. Handy for testing whether JIT'd code +// and JIT symbol tables agree on addresses. + + .section __TEXT,__text,regular,pure_instructions + .build_version macos, 14, 0 + .globl _ret_self + .p2align 4, 0x90 +_ret_self: + leaq _ret_self(%rip), %rax + retq + +.subsections_via_symbols diff --git a/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-dlsym.c b/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-dlsym.c new file mode 100644 index 0000000000000000000000000000000000000000..b597f5dc2fe68a4c90b37dc18263124376e3b865 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-dlsym.c @@ -0,0 +1,44 @@ +// Test that __orc_rt_macho_jit_dlsym works as expected. +// +// RUN: %clang -c -o %t.sym.o %p/Inputs/ret_self.S +// RUN: %clang -c -o %t.test.o %s +// RUN: %llvm_jitlink \ +// RUN: -alias Platform:_dlopen=___orc_rt_macho_jit_dlopen \ +// RUN: -alias Platform:_dlsym=___orc_rt_macho_jit_dlsym \ +// RUN: -alias Platform:_dlclose=___orc_rt_macho_jit_dlclose \ +// RUN: %t.test.o -lextra_sym -jd extra_sym %t.sym.o | FileCheck %s + +// CHECK: entering main +// CHECK-NEXT: found "ret_self" at +// CHECK-NEXT: address of "ret_self" is consistent +// CHECK-NEXT: leaving main + +int printf(const char *restrict format, ...); +void *dlopen(const char *path, int mode); +void *dlsym(void *handle, const char *symbol); +int dlclose(void *handle); + +int main(int argc, char *argv[]) { + printf("entering main\n"); + void *H = dlopen("extra_sym", 0); + if (!H) { + printf("failed\n"); + return -1; + } + + void *(*ret_self)(void) = (void *(*)(void))dlsym(H, "ret_self"); + if (ret_self) + printf("found \"ret_self\" at %p\n", ret_self); + else + printf("failed to find \"ret_self\" via dlsym\n"); + + printf("address of \"ret_self\" is %s\n", + ret_self() == ret_self ? "consistent" : "inconsistent"); + + if (dlclose(H) == -1) { + printf("failed\n"); + return -1; + } + printf("leaving main\n"); + return 0; +} diff --git a/compiler-rt/test/profile/ContinuousSyncMode/reset-default-profile.c b/compiler-rt/test/profile/ContinuousSyncMode/reset-default-profile.c new file mode 100644 index 0000000000000000000000000000000000000000..75af7684161c9b403e0ba1ca18898b94e088c3bd --- /dev/null +++ b/compiler-rt/test/profile/ContinuousSyncMode/reset-default-profile.c @@ -0,0 +1,21 @@ +// REQUIRES: darwin || linux + +// Test when LLVM_PROFILE_FILE is set incorrectly, it should fall backs to use default.profraw without runtime error. + +// Create & cd into a temporary directory. +// RUN: rm -rf %t.dir && mkdir -p %t.dir && cd %t.dir +// RUN: %clang -fprofile-instr-generate -fcoverage-mapping -mllvm -runtime-counter-relocation=true -o %t.exe %s +// RUN: env LLVM_PROFILE_FILE="incorrect-profile-name%m%c%c.profraw" %run %t.exe +// RUN: ls -l | FileCheck %s + +// CHECK: default.profraw +// CHECK-NOT: incorrect-profile-name.profraw + +#include +int f() { return 0; } + +int main(int argc, char **argv) { + FILE *File = fopen("default.profraw", "w"); + f(); + return 0; +} diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py index 50a97e2b77d15dc5cb56911d79519493f86c91ca..0e20cfbbd264b232a08d54db414aad8546c7149f 100644 --- a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py +++ b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py @@ -276,6 +276,13 @@ class VisualStudio( project.Properties, "ActiveConfiguration" ).Object ActiveConfiguration.DebugSettings.CommandArguments = cmdline_str + ConfigurationName = ActiveConfiguration.ConfigurationName + SolConfig = self._fetch_property( + self._interface.Solution.SolutionBuild.SolutionConfigurations, + ConfigurationName, + ) + for Context in SolConfig.SolutionContexts: + Context.ShouldBuild = False self.context.logger.note("Launching VS debugger...") self._fn_go(False) diff --git a/flang/include/flang/Runtime/character.h b/flang/include/flang/Runtime/character.h index 24f26920bdd2cc909c74c7a1f58d7455c79cba1b..768de75b639c554c93a8505099330485648d9fe9 100644 --- a/flang/include/flang/Runtime/character.h +++ b/flang/include/flang/Runtime/character.h @@ -44,17 +44,6 @@ void RTNAME(CharacterConcatenate)(Descriptor &accumulator, void RTNAME(CharacterConcatenateScalar1)( Descriptor &accumulator, const char *from, std::size_t chars); -// Copies the value(s) of 'rhs' to 'lhs'. Handles reallocation, -// truncation, or padding ss necessary. Crashes when not conforming and -// the LHS is not allocatable. Assumes independence of data. -// The LHS and RHS need not have the same kind of character; -// so when the LHS is a deallocated allocatable temporary result, this -// function can be used as a simple conversion routine. -// Call MoveAlloc() instead as an optimization when a temporary value is -// being assigned to a deferred-length allocatable. -void RTNAME(CharacterAssign)(Descriptor &lhs, const Descriptor &rhs, - const char *sourceFile = nullptr, int sourceLine = 0); - // CHARACTER comparisons. The kinds must match. Like std::memcmp(), // the result is less than zero, zero, or greater than zero if the first // argument is less than the second, equal to the second, or greater than diff --git a/flang/include/flang/Runtime/iostat.h b/flang/include/flang/Runtime/iostat.h index 0456e24f4e381ba8ec357ddea8de99bacdfb06f7..afce509cf1f564043596326edfb257903a699ec1 100644 --- a/flang/include/flang/Runtime/iostat.h +++ b/flang/include/flang/Runtime/iostat.h @@ -85,6 +85,7 @@ enum Iostat { IostatBadOpOnChildUnit, IostatBadNewUnit, IostatBadListDirectedInputSeparator, + IostatNonExternalDefinedUnformattedIo, }; const char *IostatErrorString(int); diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp index b114c552b552d0314f798c40d840297a96e39859..d4a3e164d207396ea68c22dc1102116e2549f567 100644 --- a/flang/lib/Frontend/FrontendActions.cpp +++ b/flang/lib/Frontend/FrontendActions.cpp @@ -58,6 +58,7 @@ #include "llvm/Passes/PassBuilder.h" #include "llvm/Passes/PassPlugin.h" #include "llvm/Passes/StandardInstrumentations.h" +#include "llvm/Support/AMDGPUAddrSpace.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FileSystem.h" @@ -166,12 +167,6 @@ static void addAMDGPUSpecificMLIRItems(mlir::ModuleOp &mlirModule, const llvm::Triple triple(targetOpts.triple); const llvm::StringRef codeObjectVersionGlobalOpName = "__oclc_ABI_version"; - // TODO: Share address spaces enumeration between Clang and Flang. - // Currently this enumeration is defined in Clang specific class - // defined in file: clang/lib/Basic/Targets/AMDGPU.h . - // and we need to move it to LLVM directory. - const int constantAddressSpace = 4; - if (!triple.isAMDGPU()) { return; } @@ -202,7 +197,7 @@ static void addAMDGPUSpecificMLIRItems(mlir::ModuleOp &mlirModule, originalGVOp.setValueAttr( builder.getIntegerAttr(int32Type, oclcABIVERsion)); originalGVOp.setUnnamedAddr(mlir::LLVM::UnnamedAddr::Local); - originalGVOp.setAddrSpace(constantAddressSpace); + originalGVOp.setAddrSpace(llvm::AMDGPUAS::CONSTANT_ADDRESS); originalGVOp.setVisibility_(mlir::LLVM::Visibility::Hidden); return; } @@ -213,7 +208,7 @@ static void addAMDGPUSpecificMLIRItems(mlir::ModuleOp &mlirModule, /* Name */ codeObjectVersionGlobalOpName, /* Value */ builder.getIntegerAttr(int32Type, oclcABIVERsion)); covInfo.setUnnamedAddr(mlir::LLVM::UnnamedAddr::Local); - covInfo.setAddrSpace(constantAddressSpace); + covInfo.setAddrSpace(llvm::AMDGPUAS::CONSTANT_ADDRESS); covInfo.setVisibility_(mlir::LLVM::Visibility::Hidden); builder.setInsertionPointToStart(mlirModule.getBody()); builder.insert(covInfo); @@ -715,8 +710,8 @@ void CodeGenAction::lowerHLFIRToFIR() { // TODO: We should get this from TargetInfo. However, that depends on // too much of clang, so for now, replicate the functionality. static std::optional> -getVScaleRange(CompilerInstance &ci, - const Fortran::frontend::LangOptions &langOpts) { +getVScaleRange(CompilerInstance &ci) { + const auto &langOpts = ci.getInvocation().getLangOpts(); if (langOpts.VScaleMin || langOpts.VScaleMax) return std::pair( langOpts.VScaleMin ? langOpts.VScaleMin : 1, langOpts.VScaleMax); @@ -751,13 +746,9 @@ void CodeGenAction::generateLLVMIR() { const auto targetOpts = ci.getInvocation().getTargetOpts(); const llvm::Triple triple(targetOpts.triple); - // Only get the vscale range if AArch64. - if (triple.isAArch64()) { - auto langOpts = ci.getInvocation().getLangOpts(); - if (auto vsr = getVScaleRange(ci, langOpts)) { - config.VScaleMin = vsr->first; - config.VScaleMax = vsr->second; - } + if (auto vsr = getVScaleRange(ci)) { + config.VScaleMin = vsr->first; + config.VScaleMax = vsr->second; } // Create the pass pipeline diff --git a/flang/lib/Lower/OpenMP.cpp b/flang/lib/Lower/OpenMP.cpp index 0fa1ac76d57edbf655547b3d5d03ed57e7e12878..eeba87fcd151169ac0f2abdf5ea2b16ffa6dd1b1 100644 --- a/flang/lib/Lower/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP.cpp @@ -2150,13 +2150,7 @@ static void createBodyOfOp( firOpBuilder, eval.getNestedEvaluations()); // Insert the terminator. - if constexpr (std::is_same_v || - std::is_same_v) { - mlir::ValueRange results; - firOpBuilder.create(loc, results); - } else { - firOpBuilder.create(loc); - } + Fortran::lower::genOpenMPTerminator(firOpBuilder, op.getOperation(), loc); // Reset the insert point to before the terminator. resetBeforeTerminator(firOpBuilder, storeOp, block); @@ -3233,7 +3227,6 @@ static void genOMP(Fortran::lower::AbstractConverter &converter, Fortran::lower::pft::Evaluation &eval, const Fortran::parser::OpenMPSectionConstruct §ionConstruct) { - fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); mlir::Location currentLocation = converter.getCurrentLocation(); const Fortran::parser::OpenMPConstruct *parentOmpConstruct = eval.parentConstruct->getIf(); @@ -3251,10 +3244,9 @@ genOMP(Fortran::lower::AbstractConverter &converter, .t); // Currently only private/firstprivate clause is handled, and // all privatization is done within `omp.section` operations. - mlir::omp::SectionOp sectionOp = - firOpBuilder.create(currentLocation); - createBodyOfOp(sectionOp, converter, currentLocation, - eval, §ionsClauseList); + genOpWithBody(converter, eval, currentLocation, + /*outerCombined=*/false, + §ionsClauseList); } static void diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp index 0b1e57e8f6c37f983db0b7594ac572b125b825c6..17efa45b8667d383762d90414c5e17d5726af8e2 100644 --- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp +++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp @@ -1132,8 +1132,13 @@ hlfir::genTypeAndKindConvert(mlir::Location loc, fir::FirOpBuilder &builder, std::optional toKindCharConvert; if (auto toCharTy = mlir::dyn_cast(toType)) { if (auto fromCharTy = mlir::dyn_cast(fromType)) - if (toCharTy.getFKind() != fromCharTy.getFKind()) + if (toCharTy.getFKind() != fromCharTy.getFKind()) { toKindCharConvert = toCharTy.getFKind(); + // Preserve source length (padding/truncation will occur in assignment + // if needed). + toType = fir::CharacterType::get( + fromType.getContext(), toCharTy.getFKind(), fromCharTy.getLen()); + } // Do not convert in case of character length mismatch only, hlfir.assign // deals with it. if (!toKindCharConvert) diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp index bf175c8ebadee6768338c5befb8cd9faecf27444..293208ce3b601a77066c4f5bf62db9763274622a 100644 --- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp +++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp @@ -2387,6 +2387,9 @@ struct XArrayCoorOpConversion const bool baseIsBoxed = coor.getMemref().getType().isa(); TypePair baseBoxTyPair = baseIsBoxed ? getBoxTypePair(coor.getMemref().getType()) : TypePair{}; + mlir::LLVM::IntegerOverflowFlagsAttr nsw = + mlir::LLVM::IntegerOverflowFlagsAttr::get( + rewriter.getContext(), mlir::LLVM::IntegerOverflowFlags::nsw); // For each dimension of the array, generate the offset calculation. for (unsigned i = 0; i < rank; ++i, ++indexOffset, ++shapeOffset, @@ -2407,14 +2410,15 @@ struct XArrayCoorOpConversion if (normalSlice) step = integerCast(loc, rewriter, idxTy, operands[sliceOffset + 2]); } - auto idx = rewriter.create(loc, idxTy, index, lb); + auto idx = rewriter.create(loc, idxTy, index, lb, nsw); mlir::Value diff = - rewriter.create(loc, idxTy, idx, step); + rewriter.create(loc, idxTy, idx, step, nsw); if (normalSlice) { mlir::Value sliceLb = integerCast(loc, rewriter, idxTy, operands[sliceOffset]); - auto adj = rewriter.create(loc, idxTy, sliceLb, lb); - diff = rewriter.create(loc, idxTy, diff, adj); + auto adj = + rewriter.create(loc, idxTy, sliceLb, lb, nsw); + diff = rewriter.create(loc, idxTy, diff, adj, nsw); } // Update the offset given the stride and the zero based index `diff` // that was just computed. @@ -2422,17 +2426,21 @@ struct XArrayCoorOpConversion // Use stride in bytes from the descriptor. mlir::Value stride = getStrideFromBox(loc, baseBoxTyPair, operands[0], i, rewriter); - auto sc = rewriter.create(loc, idxTy, diff, stride); - offset = rewriter.create(loc, idxTy, sc, offset); + auto sc = + rewriter.create(loc, idxTy, diff, stride, nsw); + offset = + rewriter.create(loc, idxTy, sc, offset, nsw); } else { // Use stride computed at last iteration. - auto sc = rewriter.create(loc, idxTy, diff, prevExt); - offset = rewriter.create(loc, idxTy, sc, offset); + auto sc = + rewriter.create(loc, idxTy, diff, prevExt, nsw); + offset = + rewriter.create(loc, idxTy, sc, offset, nsw); // Compute next stride assuming contiguity of the base array // (in element number). auto nextExt = integerCast(loc, rewriter, idxTy, operands[shapeOffset]); - prevExt = - rewriter.create(loc, idxTy, prevExt, nextExt); + prevExt = rewriter.create(loc, idxTy, prevExt, + nextExt, nsw); } } @@ -2491,8 +2499,8 @@ struct XArrayCoorOpConversion assert(coor.getLenParams().size() == 1); auto length = integerCast(loc, rewriter, idxTy, operands[coor.lenParamsOffset()]); - offset = - rewriter.create(loc, idxTy, offset, length); + offset = rewriter.create(loc, idxTy, offset, + length, nsw); } else { TODO(loc, "compute size of derived type with type parameters"); } @@ -2665,6 +2673,9 @@ private: auto cpnTy = fir::dyn_cast_ptrOrBoxEleTy(boxObjTy); mlir::Type llvmPtrTy = ::getLlvmPtrType(coor.getContext()); mlir::Type byteTy = ::getI8Type(coor.getContext()); + mlir::LLVM::IntegerOverflowFlagsAttr nsw = + mlir::LLVM::IntegerOverflowFlagsAttr::get( + rewriter.getContext(), mlir::LLVM::IntegerOverflowFlags::nsw); for (unsigned i = 1, last = operands.size(); i < last; ++i) { if (auto arrTy = cpnTy.dyn_cast()) { @@ -2680,9 +2691,9 @@ private: index < lastIndex; ++index) { mlir::Value stride = getStrideFromBox(loc, boxTyPair, operands[0], index - i, rewriter); - auto sc = rewriter.create(loc, idxTy, - operands[index], stride); - off = rewriter.create(loc, idxTy, sc, off); + auto sc = rewriter.create( + loc, idxTy, operands[index], stride, nsw); + off = rewriter.create(loc, idxTy, sc, off, nsw); } resultAddr = rewriter.create( loc, llvmPtrTy, byteTy, resultAddr, diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp index 449ea60144424a84e7329f6143b5b0b30e7db1d2..79cdaccf1fbfec180f277453e8a4a1fd9298dc19 100644 --- a/flang/lib/Parser/prescan.cpp +++ b/flang/lib/Parser/prescan.cpp @@ -706,6 +706,7 @@ void Prescanner::QuotedCharacterLiteral( char quote{*at_}; const char *end{at_ + 1}; inCharLiteral_ = true; + continuationInCharLiteral_ = true; const auto emit{[&](char ch) { EmitChar(tokens, ch); }}; const auto insert{[&](char ch) { EmitInsertedChar(tokens, ch); }}; bool isEscaped{false}; @@ -749,16 +750,9 @@ void Prescanner::QuotedCharacterLiteral( break; } inCharLiteral_ = true; - if (insertASpace_) { - if (features_.ShouldWarn( - common::LanguageFeature::MiscSourceExtensions)) { - Say(GetProvenanceRange(at_, end), - "Repeated quote mark in character literal continuation line should have been preceded by '&'"_port_en_US); - } - insertASpace_ = false; - } } } + continuationInCharLiteral_ = false; inCharLiteral_ = false; } @@ -1122,7 +1116,15 @@ const char *Prescanner::FreeFormContinuationLine(bool ampersand) { } else if (*p == '!' || *p == '\n' || *p == '#') { return nullptr; } else if (ampersand || IsImplicitContinuation()) { - if (p > nextLine_) { + if (continuationInCharLiteral_) { + // 'a'& -> 'a''b' == "a'b" + // 'b' + if (features_.ShouldWarn( + common::LanguageFeature::MiscSourceExtensions)) { + Say(GetProvenanceRange(p, p + 1), + "Character literal continuation line should have been preceded by '&'"_port_en_US); + } + } else if (p > nextLine_) { --p; } else { insertASpace_ = true; diff --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h index 16b2c6165f611c77579963c28fa8f24b0ab74322..84e046c1b102f0a5c397165038b68d8595a387fb 100644 --- a/flang/lib/Parser/prescan.h +++ b/flang/lib/Parser/prescan.h @@ -218,6 +218,7 @@ private: bool slashInCurrentStatement_{false}; bool preventHollerith_{false}; // CHARACTER*4HIMOM not Hollerith bool inCharLiteral_{false}; + bool continuationInCharLiteral_{false}; bool inPreprocessorDirective_{false}; // In some edge cases of compiler directive continuation lines, it diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp index 2f4eb9a8542700e9de54896d2cf8e24322ff929a..c430375d5ed011b48c8b1b31f094b3ba29a220b6 100644 --- a/flang/lib/Semantics/check-omp-structure.cpp +++ b/flang/lib/Semantics/check-omp-structure.cpp @@ -2996,11 +2996,14 @@ void OmpStructureChecker::Enter(const parser::OmpClause::IsDevicePtr &x) { source.ToString()); } else if (!(IsDummy(*symbol))) { context_.Say(itr->second->source, - "Variable '%s' in IS_DEVICE_PTR clause must be a dummy argument"_err_en_US, + "Variable '%s' in IS_DEVICE_PTR clause must be a dummy argument. " + "This semantic check is deprecated from OpenMP 5.2 and later."_warn_en_US, source.ToString()); } else if (IsAllocatableOrPointer(*symbol) || IsValue(*symbol)) { context_.Say(itr->second->source, - "Variable '%s' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute."_err_en_US, + "Variable '%s' in IS_DEVICE_PTR clause must be a dummy argument " + "that does not have the ALLOCATABLE, POINTER or VALUE attribute. " + "This semantic check is deprecated from OpenMP 5.2 and later."_warn_en_US, source.ToString()); } } diff --git a/flang/lib/Semantics/definable.cpp b/flang/lib/Semantics/definable.cpp index d5ffcabc7233ca9084ea7031e39e5266b5da7ce1..b73290109248a01a9ea7af045972ce60d6f28469 100644 --- a/flang/lib/Semantics/definable.cpp +++ b/flang/lib/Semantics/definable.cpp @@ -89,7 +89,8 @@ static const Symbol &GetRelevantSymbol(const evaluate::DataRef &dataRef, // Check the leftmost (or only) symbol from a data-ref or expression. static std::optional WhyNotDefinableBase(parser::CharBlock at, - const Scope &scope, DefinabilityFlags flags, const Symbol &original) { + const Scope &scope, DefinabilityFlags flags, const Symbol &original, + bool isWholeSymbol) { const Symbol &ultimate{original.GetUltimate()}; bool isPointerDefinition{flags.test(DefinabilityFlag::PointerDefinition)}; bool acceptAllocatable{flags.test(DefinabilityFlag::AcceptAllocatable)}; @@ -104,7 +105,8 @@ static std::optional WhyNotDefinableBase(parser::CharBlock at, } else if (auto dataRef{evaluate::ExtractDataRef( *association->expr(), true, true)}) { return WhyNotDefinableBase(at, scope, flags, - GetRelevantSymbol(*dataRef, isPointerDefinition, acceptAllocatable)); + GetRelevantSymbol(*dataRef, isPointerDefinition, acceptAllocatable), + isWholeSymbol); } } if (isTargetDefinition) { @@ -112,7 +114,8 @@ static std::optional WhyNotDefinableBase(parser::CharBlock at, return BlameSymbol(at, "'%s' is not a variable"_en_US, original); } else if (IsProtected(ultimate) && IsUseAssociated(original, scope)) { return BlameSymbol(at, "'%s' is protected in this scope"_en_US, original); - } else if (IsIntentIn(ultimate)) { + } else if (IsIntentIn(ultimate) && + (!IsPointer(ultimate) || (isWholeSymbol && isPointerDefinition))) { return BlameSymbol( at, "'%s' is an INTENT(IN) dummy argument"_en_US, original); } @@ -165,6 +168,12 @@ static std::optional WhyNotDefinableBase(parser::CharBlock at, static std::optional WhyNotDefinableLast(parser::CharBlock at, const Scope &scope, DefinabilityFlags flags, const Symbol &original) { const Symbol &ultimate{original.GetUltimate()}; + if (const auto *association{ultimate.detailsIf()}) { + if (auto dataRef{ + evaluate::ExtractDataRef(*association->expr(), true, true)}) { + return WhyNotDefinableLast(at, scope, flags, dataRef->GetLastSymbol()); + } + } if (flags.test(DefinabilityFlag::PointerDefinition)) { if (flags.test(DefinabilityFlag::AcceptAllocatable)) { if (!IsAllocatableOrObjectPointer(&ultimate)) { @@ -216,7 +225,8 @@ static std::optional WhyNotDefinable(parser::CharBlock at, const Symbol &base{GetRelevantSymbol(dataRef, flags.test(DefinabilityFlag::PointerDefinition), flags.test(DefinabilityFlag::AcceptAllocatable))}; - if (auto whyNot{WhyNotDefinableBase(at, scope, flags, base)}) { + if (auto whyNot{WhyNotDefinableBase(at, scope, flags, base, + std::holds_alternative(dataRef.u))}) { return whyNot; } else { return WhyNotDefinableLast(at, scope, flags, dataRef.GetLastSymbol()); @@ -231,12 +241,13 @@ static std::optional WhyNotDefinable(parser::CharBlock at, const Symbol &base{GetRelevantSymbol(dataRef, false, false)}; DefinabilityFlags baseFlags{flags}; baseFlags.reset(DefinabilityFlag::PointerDefinition); - return WhyNotDefinableBase(at, scope, baseFlags, base); + return WhyNotDefinableBase(at, scope, baseFlags, base, + std::holds_alternative(dataRef.u)); } std::optional WhyNotDefinable(parser::CharBlock at, const Scope &scope, DefinabilityFlags flags, const Symbol &original) { - if (auto base{WhyNotDefinableBase(at, scope, flags, original)}) { + if (auto base{WhyNotDefinableBase(at, scope, flags, original, true)}) { return base; } return WhyNotDefinableLast(at, scope, flags, original); diff --git a/flang/runtime/character.cpp b/flang/runtime/character.cpp index 571f64ec42f42cb1b898a64ffd989ebf11f916de..2afde7cd5e833afdf5826344aa5bbd46f4df077f 100644 --- a/flang/runtime/character.cpp +++ b/flang/runtime/character.cpp @@ -629,148 +629,6 @@ void RTNAME(CharacterConcatenateScalar1)( FreeMemory(old); } -void RTNAME(CharacterAssign)(Descriptor &lhs, const Descriptor &rhs, - const char *sourceFile, int sourceLine) { - Terminator terminator{sourceFile, sourceLine}; - int rank{lhs.rank()}; - RUNTIME_CHECK(terminator, rhs.rank() == 0 || rhs.rank() == rank); - SubscriptValue ub[maxRank], lhsAt[maxRank], rhsAt[maxRank]; - SubscriptValue elements{1}; - std::size_t lhsBytes{lhs.ElementBytes()}; - std::size_t rhsBytes{rhs.ElementBytes()}; - bool reallocate{lhs.IsAllocatable() && - (lhs.raw().base_addr == nullptr || lhsBytes != rhsBytes)}; - for (int j{0}; j < rank; ++j) { - lhsAt[j] = lhs.GetDimension(j).LowerBound(); - if (rhs.rank() > 0) { - SubscriptValue lhsExt{lhs.GetDimension(j).Extent()}; - SubscriptValue rhsExt{rhs.GetDimension(j).Extent()}; - ub[j] = lhsAt[j] + rhsExt - 1; - if (lhsExt != rhsExt) { - if (lhs.IsAllocatable()) { - reallocate = true; - } else { - terminator.Crash("Character array assignment: operands are not " - "conforming on dimension %d (%jd != %jd)", - j + 1, static_cast(lhsExt), - static_cast(rhsExt)); - } - } - rhsAt[j] = rhs.GetDimension(j).LowerBound(); - } else { - ub[j] = lhs.GetDimension(j).UpperBound(); - } - elements *= ub[j] - lhsAt[j] + 1; - } - void *old{nullptr}; - if (reallocate) { - old = lhs.raw().base_addr; - lhs.set_base_addr(nullptr); - lhs.raw().elem_len = lhsBytes = rhsBytes; - if (rhs.rank() > 0) { - // When the RHS is not scalar, the LHS acquires its bounds. - for (int j{0}; j < rank; ++j) { - lhsAt[j] = rhsAt[j]; - ub[j] = rhs.GetDimension(j).UpperBound(); - lhs.GetDimension(j).SetBounds(lhsAt[j], ub[j]); - } - } - RUNTIME_CHECK(terminator, lhs.Allocate() == CFI_SUCCESS); - } - switch (lhs.raw().type) { - case CFI_type_char: - switch (rhs.raw().type) { - case CFI_type_char: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), lhsBytes, - rhsBytes); - } - break; - case CFI_type_char16_t: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes, rhsBytes >> 1); - } - break; - case CFI_type_char32_t: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes, rhsBytes >> 2); - } - break; - default: - terminator.Crash( - "RHS of character assignment does not have a character type"); - } - break; - case CFI_type_char16_t: - switch (rhs.raw().type) { - case CFI_type_char: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes >> 1, rhsBytes); - } - break; - case CFI_type_char16_t: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes >> 1, rhsBytes >> 1); - } - break; - case CFI_type_char32_t: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes >> 1, rhsBytes >> 2); - } - break; - default: - terminator.Crash( - "RHS of character assignment does not have a character type"); - } - break; - case CFI_type_char32_t: - switch (rhs.raw().type) { - case CFI_type_char: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes >> 2, rhsBytes); - } - break; - case CFI_type_char16_t: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes >> 2, rhsBytes >> 1); - } - break; - case CFI_type_char32_t: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes >> 2, rhsBytes >> 2); - } - break; - default: - terminator.Crash( - "RHS of character assignment does not have a character type"); - } - break; - default: - terminator.Crash( - "LHS of character assignment does not have a character type"); - } - if (reallocate) { - FreeMemory(old); - } -} - int RTNAME(CharacterCompareScalar)(const Descriptor &x, const Descriptor &y) { Terminator terminator{__FILE__, __LINE__}; RUNTIME_CHECK(terminator, x.rank() == 0); diff --git a/flang/runtime/descriptor-io.cpp b/flang/runtime/descriptor-io.cpp index 563a69e999d5f45e4d8d1fcca2e481f1745971c5..6041104773cc490e492b5c58fab019bff94a6e49 100644 --- a/flang/runtime/descriptor-io.cpp +++ b/flang/runtime/descriptor-io.cpp @@ -14,7 +14,8 @@ namespace Fortran::runtime::io::descr { // Defined formatted I/O (maybe) std::optional DefinedFormattedIo(IoStatementState &io, const Descriptor &descriptor, const typeInfo::DerivedType &derived, - const typeInfo::SpecialBinding &special) { + const typeInfo::SpecialBinding &special, + const SubscriptValue subscripts[]) { std::optional peek{io.GetNextDataEdit(0 /*to peek at it*/)}; if (peek && (peek->descriptor == DataEdit::DefinedDerivedType || @@ -61,9 +62,6 @@ std::optional DefinedFormattedIo(IoStatementState &io, // I/O subroutine reads counts towards READ(SIZE=). startPos = io.InquirePos(); } - std::size_t numElements{descriptor.Elements()}; - SubscriptValue subscripts[maxRank]; - descriptor.GetLowerBounds(subscripts); if (special.IsArgDescriptor(0)) { // "dtv" argument is "class(t)", pass a descriptor auto *p{special.GetProc DefinedFormattedIo(IoStatementState &io, Descriptor &elementDesc{elementStatDesc.descriptor()}; elementDesc.Establish( derived, nullptr, 0, nullptr, CFI_attribute_pointer); - for (; numElements-- > 0; descriptor.IncrementSubscripts(subscripts)) { - elementDesc.set_base_addr(descriptor.Element(subscripts)); - p(elementDesc, unit, ioType, vListDesc, ioStat, ioMsg, ioTypeLen, - sizeof ioMsg); - if (ioStat != IostatOk) { - break; - } - } + elementDesc.set_base_addr(descriptor.Element(subscripts)); + p(elementDesc, unit, ioType, vListDesc, ioStat, ioMsg, ioTypeLen, + sizeof ioMsg); } else { // "dtv" argument is "type(t)", pass a raw pointer auto *p{special.GetProc()}; - for (; numElements-- > 0; descriptor.IncrementSubscripts(subscripts)) { - p(descriptor.Element(subscripts), unit, ioType, vListDesc, ioStat, - ioMsg, ioTypeLen, sizeof ioMsg); - if (ioStat != IostatOk) { - break; - } - } + p(descriptor.Element(subscripts), unit, ioType, vListDesc, ioStat, + ioMsg, ioTypeLen, sizeof ioMsg); } handler.Forward(ioStat, ioMsg, sizeof ioMsg); external->PopChildIo(child); @@ -119,7 +107,10 @@ bool DefinedUnformattedIo(IoStatementState &io, const Descriptor &descriptor, // Unformatted I/O must have an external unit (or child thereof). IoErrorHandler &handler{io.GetIoErrorHandler()}; ExternalFileUnit *external{io.GetExternalFileUnit()}; - RUNTIME_CHECK(handler, external != nullptr); + if (!external) { // INQUIRE(IOLENGTH=) + handler.SignalError(IostatNonExternalDefinedUnformattedIo); + return false; + } ChildIo &child{external->PushChildIo(io)}; int unit{external->unitNumber()}; int ioStat{IostatOk}; diff --git a/flang/runtime/descriptor-io.h b/flang/runtime/descriptor-io.h index 2b5bf8248aca2a3de6f85e29a8f68e9683d7d64a..394578796faa7928b5bfd7b54fc74c3dd1586fd6 100644 --- a/flang/runtime/descriptor-io.h +++ b/flang/runtime/descriptor-io.h @@ -268,7 +268,33 @@ static bool DefaultComponentIO(IoStatementState &io, } template -static bool DefaultComponentwiseIO(IoStatementState &io, +static bool DefaultComponentwiseFormattedIO(IoStatementState &io, + const Descriptor &descriptor, const typeInfo::DerivedType &type, + const NonTbpDefinedIoTable *table, const SubscriptValue subscripts[]) { + IoErrorHandler &handler{io.GetIoErrorHandler()}; + const Descriptor &compArray{type.component()}; + RUNTIME_CHECK(handler, compArray.rank() == 1); + std::size_t numComponents{compArray.Elements()}; + SubscriptValue at[maxRank]; + compArray.GetLowerBounds(at); + for (std::size_t k{0}; k < numComponents; + ++k, compArray.IncrementSubscripts(at)) { + const typeInfo::Component &component{ + *compArray.Element(at)}; + if (!DefaultComponentIO( + io, component, descriptor, subscripts, handler, table)) { + // Return true for NAMELIST input if any component appeared. + auto *listInput{ + io.get_if>()}; + return DIR == Direction::Input && k > 0 && listInput && + listInput->inNamelistSequence(); + } + } + return true; +} + +template +static bool DefaultComponentwiseUnformattedIO(IoStatementState &io, const Descriptor &descriptor, const typeInfo::DerivedType &type, const NonTbpDefinedIoTable *table) { IoErrorHandler &handler{io.GetIoErrorHandler()}; @@ -288,11 +314,7 @@ static bool DefaultComponentwiseIO(IoStatementState &io, *compArray.Element(at)}; if (!DefaultComponentIO( io, component, descriptor, subscripts, handler, table)) { - // Truncated nonempty namelist input sequence? - auto *listInput{ - io.get_if>()}; - return DIR == Direction::Input && (j > 0 || k > 0) && listInput && - listInput->inNamelistSequence(); + return false; } } } @@ -300,7 +322,8 @@ static bool DefaultComponentwiseIO(IoStatementState &io, } std::optional DefinedFormattedIo(IoStatementState &, const Descriptor &, - const typeInfo::DerivedType &, const typeInfo::SpecialBinding &); + const typeInfo::DerivedType &, const typeInfo::SpecialBinding &, + const SubscriptValue[]); template static bool FormattedDerivedTypeIO(IoStatementState &io, @@ -311,37 +334,54 @@ static bool FormattedDerivedTypeIO(IoStatementState &io, RUNTIME_CHECK(handler, addendum != nullptr); const typeInfo::DerivedType *type{addendum->derivedType()}; RUNTIME_CHECK(handler, type != nullptr); + std::optional nonTbpSpecial; + const typeInfo::SpecialBinding *special{nullptr}; if (table) { if (const auto *definedIo{table->Find(*type, DIR == Direction::Input ? common::DefinedIo::ReadFormatted : common::DefinedIo::WriteFormatted)}) { if (definedIo->subroutine) { - typeInfo::SpecialBinding special{DIR == Direction::Input + nonTbpSpecial.emplace(DIR == Direction::Input ? typeInfo::SpecialBinding::Which::ReadFormatted : typeInfo::SpecialBinding::Which::WriteFormatted, definedIo->subroutine, definedIo->isDtvArgPolymorphic, false, - false}; - if (std::optional wasDefined{ - DefinedFormattedIo(io, descriptor, *type, special)}) { - return *wasDefined; - } - } else { - return DefaultComponentwiseIO(io, descriptor, *type, table); + false); + special = &*nonTbpSpecial; } } } - if (const typeInfo::SpecialBinding * - special{type->FindSpecialBinding(DIR == Direction::Input - ? typeInfo::SpecialBinding::Which::ReadFormatted - : typeInfo::SpecialBinding::Which::WriteFormatted)}) { - if (!table || !table->ignoreNonTbpEntries || special->isTypeBound()) { - if (std::optional wasDefined{ - DefinedFormattedIo(io, descriptor, *type, *special)}) { - return *wasDefined; // defined I/O was applied + if (!special) { + if (const typeInfo::SpecialBinding * + binding{type->FindSpecialBinding(DIR == Direction::Input + ? typeInfo::SpecialBinding::Which::ReadFormatted + : typeInfo::SpecialBinding::Which::WriteFormatted)}) { + if (!table || !table->ignoreNonTbpEntries || binding->isTypeBound()) { + special = binding; } } } - return DefaultComponentwiseIO(io, descriptor, *type, table); + SubscriptValue subscripts[maxRank]; + descriptor.GetLowerBounds(subscripts); + std::size_t numElements{descriptor.Elements()}; + for (std::size_t j{0}; j < numElements; + ++j, descriptor.IncrementSubscripts(subscripts)) { + std::optional result; + if (special) { + result = DefinedFormattedIo(io, descriptor, *type, *special, subscripts); + } + if (!result) { + result = DefaultComponentwiseFormattedIO( + io, descriptor, *type, table, subscripts); + } + if (!result.value()) { + // Return true for NAMELIST input if we got anything. + auto *listInput{ + io.get_if>()}; + return DIR == Direction::Input && j > 0 && listInput && + listInput->inNamelistSequence(); + } + } + return true; } bool DefinedUnformattedIo(IoStatementState &, const Descriptor &, @@ -371,7 +411,8 @@ static bool UnformattedDescriptorIO(IoStatementState &io, return *wasDefined; } } else { - return DefaultComponentwiseIO(io, descriptor, *type, table); + return DefaultComponentwiseUnformattedIO( + io, descriptor, *type, table); } } } @@ -388,7 +429,7 @@ static bool UnformattedDescriptorIO(IoStatementState &io, // TODO: If no component at any level has defined READ or WRITE // (as appropriate), the elements are contiguous, and no byte swapping // is active, do a block transfer via the code below. - return DefaultComponentwiseIO(io, descriptor, *type, table); + return DefaultComponentwiseUnformattedIO(io, descriptor, *type, table); } else { // intrinsic type unformatted I/O auto *externalUnf{io.get_if>()}; diff --git a/flang/runtime/edit-input.cpp b/flang/runtime/edit-input.cpp index 4e8c9aa868a691caaa84d2de7ea0aed311f0b7e9..822099b5141b1b4d86f11742d8bd9cd7a9c8d10e 100644 --- a/flang/runtime/edit-input.cpp +++ b/flang/runtime/edit-input.cpp @@ -64,10 +64,15 @@ static bool EditBOZInput( } // Count significant digits after any leading white space & zeroes int digits{0}; + int significantBits{0}; for (; next; next = io.NextInField(remaining, edit)) { char32_t ch{*next}; if (ch == ' ' || ch == '\t') { - continue; + if (edit.modes.editingFlags & blankZero) { + ch = '0'; // BZ mode - treat blank as if it were zero + } else { + continue; + } } if (ch >= '0' && ch <= '1') { } else if (LOG2_BASE >= 3 && ch >= '2' && ch <= '7') { @@ -79,9 +84,22 @@ static bool EditBOZInput( "Bad character '%lc' in B/O/Z input field", ch); return false; } - ++digits; + if (digits++ == 0) { + significantBits = 4; + if (ch >= '0' && ch <= '1') { + significantBits = 1; + } else if (ch >= '2' && ch <= '3') { + significantBits = 2; + } else if (ch >= '4' && ch <= '7') { + significantBits = 3; + } else { + significantBits = 4; + } + } else { + significantBits += LOG2_BASE; + } } - auto significantBytes{static_cast(digits * LOG2_BASE + 7) / 8}; + auto significantBytes{static_cast(significantBits + 7) / 8}; if (significantBytes > bytes) { io.GetIoErrorHandler().SignalError(IostatBOZInputOverflow, "B/O/Z input of %d digits overflows %zd-byte variable", digits, bytes); @@ -96,12 +114,17 @@ static bool EditBOZInput( auto *data{reinterpret_cast(n) + (isHostLittleEndian ? significantBytes - 1 : 0)}; int shift{((digits - 1) * LOG2_BASE) & 7}; - if (shift + LOG2_BASE > 8) { - shift -= 8; // misaligned octal - } while (digits > 0) { char32_t ch{*io.NextInField(remaining, edit)}; int digit{0}; + if (ch == ' ' || ch == '\t') { + if (edit.modes.editingFlags & blankZero) { + ch = '0'; // BZ mode - treat blank as if it were zero + } else { + continue; + } + } + --digits; if (ch >= '0' && ch <= '9') { digit = ch - '0'; } else if (ch >= 'A' && ch <= 'F') { @@ -111,12 +134,11 @@ static bool EditBOZInput( } else { continue; } - --digits; if (shift < 0) { - shift += 8; - if (shift + LOG2_BASE > 8) { // misaligned octal - *data |= digit >> (8 - shift); + if (shift + LOG2_BASE > 0) { // misaligned octal + *data |= digit >> -shift; } + shift += 8; data += increment; } *data |= digit << shift; @@ -894,20 +916,20 @@ static bool EditListDirectedCharacterInput( } template -bool EditCharacterInput( - IoStatementState &io, const DataEdit &edit, CHAR *x, std::size_t length) { +bool EditCharacterInput(IoStatementState &io, const DataEdit &edit, CHAR *x, + std::size_t lengthChars) { switch (edit.descriptor) { case DataEdit::ListDirected: - return EditListDirectedCharacterInput(io, x, length, edit); + return EditListDirectedCharacterInput(io, x, lengthChars, edit); case 'A': case 'G': break; case 'B': - return EditBOZInput<1>(io, edit, x, length * sizeof *x); + return EditBOZInput<1>(io, edit, x, lengthChars * sizeof *x); case 'O': - return EditBOZInput<3>(io, edit, x, length * sizeof *x); + return EditBOZInput<3>(io, edit, x, lengthChars * sizeof *x); case 'Z': - return EditBOZInput<4>(io, edit, x, length * sizeof *x); + return EditBOZInput<4>(io, edit, x, lengthChars * sizeof *x); default: io.GetIoErrorHandler().SignalError(IostatErrorInFormat, "Data edit descriptor '%c' may not be used with a CHARACTER data item", @@ -915,27 +937,31 @@ bool EditCharacterInput( return false; } const ConnectionState &connection{io.GetConnectionState()}; - std::size_t remaining{length}; + std::size_t remainingChars{lengthChars}; + // Skip leading characters. + // Their bytes don't count towards INQUIRE(IOLENGTH=). + std::size_t skipChars{0}; if (edit.width && *edit.width > 0) { - remaining = *edit.width; + remainingChars = *edit.width; + if (remainingChars > lengthChars) { + skipChars = remainingChars - lengthChars; + } } // When the field is wider than the variable, we drop the leading // characters. When the variable is wider than the field, there can be // trailing padding or an EOR condition. const char *input{nullptr}; - std::size_t ready{0}; - // Skip leading bytes. - // These bytes don't count towards INQUIRE(IOLENGTH=). - std::size_t skip{remaining > length ? remaining - length : 0}; + std::size_t readyBytes{0}; // Transfer payload bytes; these do count. - while (remaining > 0) { - if (ready == 0) { - ready = io.GetNextInputBytes(input); - if (ready == 0 || (ready < remaining && edit.modes.nonAdvancing)) { - if (io.CheckForEndOfRecord(ready)) { - if (ready == 0) { + while (remainingChars > 0) { + if (readyBytes == 0) { + readyBytes = io.GetNextInputBytes(input); + if (readyBytes == 0 || + (readyBytes < remainingChars && edit.modes.nonAdvancing)) { + if (io.CheckForEndOfRecord(readyBytes)) { + if (readyBytes == 0) { // PAD='YES' and no more data - std::fill_n(x, length, ' '); + std::fill_n(x, lengthChars, ' '); return !io.GetIoErrorHandler().InError(); } else { // Do partial read(s) then pad on last iteration @@ -945,63 +971,64 @@ bool EditCharacterInput( } } } - std::size_t chunk; - bool skipping{skip > 0}; + std::size_t chunkBytes; + std::size_t chunkChars{1}; + bool skipping{skipChars > 0}; if (connection.isUTF8) { - chunk = MeasureUTF8Bytes(*input); + chunkBytes = MeasureUTF8Bytes(*input); if (skipping) { - --skip; + --skipChars; } else if (auto ucs{DecodeUTF8(input)}) { *x++ = *ucs; - --length; - } else if (chunk == 0) { + --lengthChars; + } else if (chunkBytes == 0) { // error recovery: skip bad encoding - chunk = 1; + chunkBytes = 1; } - --remaining; } else if (connection.internalIoCharKind > 1) { // Reading from non-default character internal unit - chunk = connection.internalIoCharKind; + chunkBytes = connection.internalIoCharKind; if (skipping) { - --skip; + --skipChars; } else { char32_t buffer{0}; - std::memcpy(&buffer, input, chunk); + std::memcpy(&buffer, input, chunkBytes); *x++ = buffer; - --length; + --lengthChars; } - --remaining; } else if constexpr (sizeof *x > 1) { // Read single byte with expansion into multi-byte CHARACTER - chunk = 1; + chunkBytes = 1; if (skipping) { - --skip; + --skipChars; } else { *x++ = static_cast(*input); - --length; + --lengthChars; } - --remaining; } else { // single bytes -> default CHARACTER if (skipping) { - chunk = std::min(skip, ready); - skip -= chunk; + chunkBytes = std::min(skipChars, readyBytes); + chunkChars = chunkBytes; + skipChars -= chunkChars; } else { - chunk = std::min(remaining, ready); - std::memcpy(x, input, chunk); - x += chunk; - length -= chunk; + chunkBytes = std::min(remainingChars, readyBytes); + chunkBytes = std::min(lengthChars, chunkBytes); + chunkChars = chunkBytes; + std::memcpy(x, input, chunkBytes); + x += chunkBytes; + lengthChars -= chunkChars; } - remaining -= chunk; } - input += chunk; + input += chunkBytes; + remainingChars -= chunkChars; if (!skipping) { - io.GotChar(chunk); + io.GotChar(chunkBytes); } - io.HandleRelativePosition(chunk); - ready -= chunk; + io.HandleRelativePosition(chunkBytes); + readyBytes -= chunkBytes; } // Pad the remainder of the input variable, if any. - std::fill_n(x, length, ' '); + std::fill_n(x, lengthChars, ' '); return CheckCompleteListDirectedField(io, edit); } diff --git a/flang/runtime/edit-output.cpp b/flang/runtime/edit-output.cpp index 18b209bc6798c33571c815df2f46423fc8031493..a4ce0b12f911116b756c6810a1a761c4a0a4e1e5 100644 --- a/flang/runtime/edit-output.cpp +++ b/flang/runtime/edit-output.cpp @@ -140,7 +140,7 @@ bool EditIntegerOutput(IoStatementState &io, const DataEdit &edit, return EditBOZOutput<4>( io, edit, reinterpret_cast(&n), KIND); case 'L': - return EditLogicalOutput(io, edit, *reinterpret_cast(&n)); + return EditLogicalOutput(io, edit, n != 0 ? true : false); case 'A': // legacy extension return EditCharacterOutput( io, edit, reinterpret_cast(&n), sizeof n); @@ -433,27 +433,28 @@ bool RealOutputEditing::EditFOutput(const DataEdit &edit) { } // Multiple conversions may be needed to get the right number of // effective rounded fractional digits. - int extraDigits{0}; bool canIncrease{true}; - while (true) { + for (int extraDigits{fracDigits == 0 ? 1 : 0};;) { decimal::ConversionToDecimalResult converted{ ConvertToDecimal(extraDigits + fracDigits, rounding, flags)}; - if (IsInfOrNaN(converted.str, static_cast(converted.length))) { + const char *convertedStr{converted.str}; + if (IsInfOrNaN(convertedStr, static_cast(converted.length))) { return editWidth > 0 && converted.length > static_cast(editWidth) ? EmitRepeated(io_, '*', editWidth) : EmitPrefix(edit, converted.length, editWidth) && - EmitAscii(io_, converted.str, converted.length) && + EmitAscii(io_, convertedStr, converted.length) && EmitSuffix(edit); } int expo{converted.decimalExponent + edit.modes.scale /*kP*/}; - int signLength{*converted.str == '-' || *converted.str == '+' ? 1 : 0}; + int signLength{*convertedStr == '-' || *convertedStr == '+' ? 1 : 0}; int convertedDigits{static_cast(converted.length) - signLength}; if (IsZero()) { // don't treat converted "0" as significant digit expo = 0; convertedDigits = 0; } - int trailingOnes{0}; + bool isNegative{*convertedStr == '-'}; + char one[2]; if (expo > extraDigits && extraDigits >= 0 && canIncrease) { extraDigits = expo; if (!edit.digits.has_value()) { // F0 @@ -462,24 +463,45 @@ bool RealOutputEditing::EditFOutput(const DataEdit &edit) { canIncrease = false; // only once continue; } else if (expo == -fracDigits && convertedDigits > 0) { - if ((rounding == decimal::FortranRounding::RoundUp && - *converted.str != '-') || - (rounding == decimal::FortranRounding::RoundDown && - *converted.str == '-') || - (rounding == decimal::FortranRounding::RoundToZero && - rounding != edit.modes.round && // it changed below - converted.str[signLength] >= '5')) { - // Round up/down to a scaled 1 + // Result will be either a signed zero or power of ten, depending + // on rounding. + char leading{convertedStr[signLength]}; + bool roundToPowerOfTen{false}; + switch (edit.modes.round) { + case decimal::FortranRounding::RoundUp: + roundToPowerOfTen = !isNegative; + break; + case decimal::FortranRounding::RoundDown: + roundToPowerOfTen = isNegative; + break; + case decimal::FortranRounding::RoundToZero: + break; + case decimal::FortranRounding::RoundNearest: + if (leading == '5' && + rounding == decimal::FortranRounding::RoundNearest) { + // Try again, rounding away from zero. + rounding = isNegative ? decimal::FortranRounding::RoundDown + : decimal::FortranRounding::RoundUp; + extraDigits = 1 - fracDigits; // just one digit needed + continue; + } + roundToPowerOfTen = leading > '5'; + break; + case decimal::FortranRounding::RoundCompatible: + roundToPowerOfTen = leading >= '5'; + break; + } + if (roundToPowerOfTen) { ++expo; - convertedDigits = 0; - trailingOnes = 1; - } else if (rounding != decimal::FortranRounding::RoundToZero) { - // Convert again with truncation so first digit can be checked - // on the next iteration by the code above - rounding = decimal::FortranRounding::RoundToZero; - continue; + convertedDigits = 1; + if (signLength > 0) { + one[0] = *convertedStr; + one[1] = '1'; + } else { + one[0] = '1'; + } + convertedStr = one; } else { - // Value rounds down to zero expo = 0; convertedDigits = 0; } @@ -493,17 +515,14 @@ bool RealOutputEditing::EditFOutput(const DataEdit &edit) { int digitsAfterPoint{convertedDigits - digitsBeforePoint}; int trailingZeroes{flags & decimal::Minimize ? 0 - : std::max(0, - fracDigits - - (zeroesAfterPoint + digitsAfterPoint + trailingOnes))}; + : std::max(0, fracDigits - (zeroesAfterPoint + digitsAfterPoint))}; if (digitsBeforePoint + zeroesBeforePoint + zeroesAfterPoint + - digitsAfterPoint + trailingOnes + trailingZeroes == + digitsAfterPoint + trailingZeroes == 0) { zeroesBeforePoint = 1; // "." -> "0." } int totalLength{signLength + digitsBeforePoint + zeroesBeforePoint + - 1 /*'.'*/ + zeroesAfterPoint + digitsAfterPoint + trailingOnes + - trailingZeroes}; + 1 /*'.'*/ + zeroesAfterPoint + digitsAfterPoint + trailingZeroes}; int width{editWidth > 0 ? editWidth : totalLength}; if (totalLength > width) { return EmitRepeated(io_, '*', width); @@ -513,13 +532,12 @@ bool RealOutputEditing::EditFOutput(const DataEdit &edit) { ++totalLength; } return EmitPrefix(edit, totalLength, width) && - EmitAscii(io_, converted.str, signLength + digitsBeforePoint) && + EmitAscii(io_, convertedStr, signLength + digitsBeforePoint) && EmitRepeated(io_, '0', zeroesBeforePoint) && EmitAscii(io_, edit.modes.editingFlags & decimalComma ? "," : ".", 1) && EmitRepeated(io_, '0', zeroesAfterPoint) && - EmitAscii(io_, converted.str + signLength + digitsBeforePoint, + EmitAscii(io_, convertedStr + signLength + digitsBeforePoint, digitsAfterPoint) && - EmitRepeated(io_, '1', trailingOnes) && EmitRepeated(io_, '0', trailingZeroes) && EmitRepeated(io_, ' ', trailingBlanks_) && EmitSuffix(edit); } diff --git a/flang/runtime/internal-unit.cpp b/flang/runtime/internal-unit.cpp index aa7130f3a6a53ddccebd699c95ca368743d61349..e3fffaa6f378fffc2df658a57e2f2dfeae88e2f9 100644 --- a/flang/runtime/internal-unit.cpp +++ b/flang/runtime/internal-unit.cpp @@ -43,11 +43,9 @@ InternalDescriptorUnit::InternalDescriptorUnit( template void InternalDescriptorUnit::EndIoStatement() { if constexpr (DIR == Direction::Output) { - // Clear the remainder of the current record if anything was written - // to it, or if it is the only record. + // Clear the remainder of the current record. auto end{endfileRecordNumber.value_or(0)}; - if (currentRecordNumber < end && - (end == 2 || furthestPositionInRecord > 0)) { + if (currentRecordNumber < end) { BlankFillOutputRecord(); } } diff --git a/flang/runtime/io-stmt.h b/flang/runtime/io-stmt.h index d4ceb83265246bd77b671469c57f0bb1b6cedd8c..91169f6c6e323e919ff51ae6c35f523d2282de9c 100644 --- a/flang/runtime/io-stmt.h +++ b/flang/runtime/io-stmt.h @@ -92,8 +92,8 @@ public: std::size_t GetNextInputBytes(const char *&); bool AdvanceRecord(int = 1); void BackspaceRecord(); - void HandleRelativePosition(std::int64_t); - void HandleAbsolutePosition(std::int64_t); // for r* in list I/O + void HandleRelativePosition(std::int64_t byteOffset); + void HandleAbsolutePosition(std::int64_t byteOffset); // for r* in list I/O std::optional GetNextDataEdit(int maxRepeat = 1); ExternalFileUnit *GetExternalFileUnit() const; // null if internal unit bool BeginReadingRecord(); @@ -124,7 +124,11 @@ public: // Vacant after the end of the current record std::optional GetCurrentChar(std::size_t &byteCount); - // For fixed-width fields, return the number of remaining characters. + // The "remaining" arguments to CueUpInput(), SkipSpaces(), & NextInField() + // are always in units of bytes, not characters; the distinction matters + // for internal input from CHARACTER(KIND=2 and 4). + + // For fixed-width fields, return the number of remaining bytes. // Skip over leading blanks. std::optional CueUpInput(const DataEdit &edit) { std::optional remaining; @@ -134,6 +138,10 @@ public: } else { if (edit.width.value_or(0) > 0) { remaining = *edit.width; + if (int bytesPerChar{GetConnectionState().internalIoCharKind}; + bytesPerChar > 1) { + *remaining *= bytesPerChar; + } } SkipSpaces(remaining); } diff --git a/flang/runtime/iostat.cpp b/flang/runtime/iostat.cpp index cc5641693a078a6e33ad6d618c9c76a46b36d8ca..c993b778e9e1f8cbf40cf38da06f43701d4f5c37 100644 --- a/flang/runtime/iostat.cpp +++ b/flang/runtime/iostat.cpp @@ -115,6 +115,8 @@ const char *IostatErrorString(int iostat) { return "NEWUNIT= without FILE= or STATUS='SCRATCH'"; case IostatBadListDirectedInputSeparator: return "List-directed input value has trailing unused characters"; + case IostatNonExternalDefinedUnformattedIo: + return "Defined unformatted I/O without an external unit"; default: return nullptr; } diff --git a/flang/runtime/unit.cpp b/flang/runtime/unit.cpp index 5fa8565c2f61fc56f3ffb080a4794b8245fdcaa5..e4f346ae941f3b8eaa84accd1da269201c2310e2 100644 --- a/flang/runtime/unit.cpp +++ b/flang/runtime/unit.cpp @@ -355,6 +355,7 @@ bool ExternalFileUnit::Emit(const char *data, std::size_t bytes, } positionInRecord += bytes; furthestPositionInRecord = furthestAfter; + anyWriteSinceLastPositioning_ = true; return true; } @@ -440,6 +441,14 @@ bool ExternalFileUnit::BeginReadingRecord(IoErrorHandler &handler) { RUNTIME_CHECK(handler, direction_ == Direction::Input); if (!beganReadingRecord_) { beganReadingRecord_ = true; + // Don't use IsAtEOF() to check for an EOF condition here, just detect + // it from a failed or short read from the file. IsAtEOF() could be + // wrong for formatted input if actual newline characters had been + // written in-band by previous WRITEs before a REWIND. In fact, + // now that we know that the unit is being used for input (again), + // it's best to reset endfileRecordNumber and ensure IsAtEOF() will + // now be true on return only if it gets set by HitEndOnRead(). + endfileRecordNumber.reset(); if (access == Access::Direct) { CheckDirectAccess(handler); auto need{static_cast(recordOffsetInFrame_ + *openRecl)}; @@ -451,18 +460,19 @@ bool ExternalFileUnit::BeginReadingRecord(IoErrorHandler &handler) { HitEndOnRead(handler); } } else { + if (anyWriteSinceLastPositioning_ && access == Access::Sequential) { + // Most Fortran implementations allow a READ after a WRITE; + // the read then just hits an EOF. + DoEndfile(handler); + } recordLength.reset(); - if (IsAtEOF()) { - handler.SignalEnd(); - } else { - RUNTIME_CHECK(handler, isUnformatted.has_value()); - if (*isUnformatted) { - if (access == Access::Sequential) { - BeginSequentialVariableUnformattedInputRecord(handler); - } - } else { // formatted sequential or stream - BeginVariableFormattedInputRecord(handler); + RUNTIME_CHECK(handler, isUnformatted.has_value()); + if (*isUnformatted) { + if (access == Access::Sequential) { + BeginSequentialVariableUnformattedInputRecord(handler); } + } else { // formatted sequential or stream + BeginVariableFormattedInputRecord(handler); } } } @@ -595,8 +605,8 @@ void ExternalFileUnit::BackspaceRecord(IoErrorHandler &handler) { if (IsAfterEndfile()) { // BACKSPACE after explicit ENDFILE currentRecordNumber = *endfileRecordNumber; - } else if (leftTabLimit) { - // BACKSPACE after non-advancing I/O + } else if (leftTabLimit && direction_ == Direction::Input) { + // BACKSPACE after non-advancing input leftTabLimit.reset(); } else { DoImpliedEndfile(handler); @@ -615,6 +625,7 @@ void ExternalFileUnit::BackspaceRecord(IoErrorHandler &handler) { } } BeginRecord(); + anyWriteSinceLastPositioning_ = false; } } @@ -667,6 +678,7 @@ void ExternalFileUnit::Rewind(IoErrorHandler &handler) { SetPosition(0, handler); currentRecordNumber = 1; leftTabLimit.reset(); + anyWriteSinceLastPositioning_ = false; } } @@ -727,6 +739,7 @@ void ExternalFileUnit::EndIoStatement() { void ExternalFileUnit::BeginSequentialVariableUnformattedInputRecord( IoErrorHandler &handler) { + RUNTIME_CHECK(handler, access == Access::Sequential); std::int32_t header{0}, footer{0}; std::size_t need{recordOffsetInFrame_ + sizeof header}; std::size_t got{ReadFrame(frameOffsetInFile_, need, handler)}; @@ -896,28 +909,29 @@ void ExternalFileUnit::BackspaceVariableFormattedRecord( } void ExternalFileUnit::DoImpliedEndfile(IoErrorHandler &handler) { - if (!impliedEndfile_ && direction_ == Direction::Output && IsRecordFile() && - access != Access::Direct && leftTabLimit) { - // Complete partial record after non-advancing write before - // positioning or closing the unit. Usually sets impliedEndfile_. - AdvanceRecord(handler); - } - if (impliedEndfile_) { - impliedEndfile_ = false; - if (access != Access::Direct && IsRecordFile() && mayPosition()) { + if (access != Access::Direct) { + if (!impliedEndfile_ && leftTabLimit && direction_ == Direction::Output) { + // Flush a partial record after non-advancing output + impliedEndfile_ = true; + } + if (impliedEndfile_ && mayPosition()) { DoEndfile(handler); } } + impliedEndfile_ = false; } void ExternalFileUnit::DoEndfile(IoErrorHandler &handler) { if (IsRecordFile() && access != Access::Direct) { furthestPositionInRecord = std::max(positionInRecord, furthestPositionInRecord); - if (leftTabLimit) { - // Last read/write was non-advancing, so AdvanceRecord() was not called. - leftTabLimit.reset(); - ++currentRecordNumber; + if (leftTabLimit) { // last I/O was non-advancing + if (access == Access::Sequential && direction_ == Direction::Output) { + AdvanceRecord(handler); + } else { // Access::Stream or input + leftTabLimit.reset(); + ++currentRecordNumber; + } } endfileRecordNumber = currentRecordNumber; } @@ -928,6 +942,7 @@ void ExternalFileUnit::DoEndfile(IoErrorHandler &handler) { TruncateFrame(frameOffsetInFile_, handler); BeginRecord(); impliedEndfile_ = false; + anyWriteSinceLastPositioning_ = false; } void ExternalFileUnit::CommitWrites() { diff --git a/flang/runtime/unit.h b/flang/runtime/unit.h index 1ec3013ba82b3fba2fac5f8554bbb7f628eabd87..140fda3c4d2a811c9c27f0e40ac41a6fe0630880 100644 --- a/flang/runtime/unit.h +++ b/flang/runtime/unit.h @@ -140,6 +140,7 @@ private: Direction direction_{Direction::Output}; bool impliedEndfile_{false}; // sequential/stream output has taken place bool beganReadingRecord_{false}; + bool anyWriteSinceLastPositioning_{false}; bool directAccessRecWasSet_{false}; // REC= appeared // Subtle: The beginning of the frame can't be allowed to advance // during a single list-directed READ due to the possibility of a diff --git a/flang/test/Driver/driver-help-hidden.f90 b/flang/test/Driver/driver-help-hidden.f90 index 8cb8b54d59412c039a39b3e34feefaab4f2a82fe..9a11a7a571ffcc6ac6879516e7b21db80417aa47 100644 --- a/flang/test/Driver/driver-help-hidden.f90 +++ b/flang/test/Driver/driver-help-hidden.f90 @@ -64,6 +64,7 @@ ! CHECK-NEXT: Select Windows run-time library ! CHECK-NEXT: -fno-automatic Implies the SAVE attribute for non-automatic local objects in subprograms unless RECURSIVE ! CHECK-NEXT: -fno-color-diagnostics Disable colors in diagnostics +! CHECK-NEXT: -fno-fortran-main Do not include Fortran_main.a (provided by Flang) when linking ! CHECK-NEXT: -fno-integrated-as Disable the integrated assembler ! CHECK-NEXT: -fno-lto Disable LTO mode (default) ! CHECK-NEXT: -fno-ppc-native-vector-element-order diff --git a/flang/test/Driver/driver-help.f90 b/flang/test/Driver/driver-help.f90 index 0607ffde23789991ac7e458d87deab7fef74447b..e0e74dc56f331ead0f3f090da71f05ae34882522 100644 --- a/flang/test/Driver/driver-help.f90 +++ b/flang/test/Driver/driver-help.f90 @@ -54,6 +54,7 @@ ! HELP-NEXT: Select Windows run-time library ! HELP-NEXT: -fno-automatic Implies the SAVE attribute for non-automatic local objects in subprograms unless RECURSIVE ! HELP-NEXT: -fno-color-diagnostics Disable colors in diagnostics +! HELP-NEXT: -fno-fortran-main Do not include Fortran_main.a (provided by Flang) when linking ! HELP-NEXT: -fno-integrated-as Disable the integrated assembler ! HELP-NEXT: -fno-lto Disable LTO mode (default) ! HELP-NEXT: -fno-ppc-native-vector-element-order diff --git a/flang/test/Driver/no-duplicate-main.f90 b/flang/test/Driver/no-duplicate-main.f90 index 4e33f4f2aeba3f703b70617f3fb1b9c02a3e3aa5..12d5e46247bad277d55262b6d503d00c3443d5f3 100644 --- a/flang/test/Driver/no-duplicate-main.f90 +++ b/flang/test/Driver/no-duplicate-main.f90 @@ -4,6 +4,8 @@ ! RUN: %flang -o %t -c %s ! RUN: not %flang -o %t.exe %t %t.c-object 2>&1 +! RUN: %flang -fno-fortran-main -o %t.exe %t %t.c-object 2>&1 + ! TODO: potentially add further checks to ensure that proper ! linker error messages are detected and checked via ! FileCheck. diff --git a/flang/test/Fir/array-coor.fir b/flang/test/Fir/array-coor.fir index 738acd7dd91fd2fcee0c3dcb39591fa0cf9cdea7..a765670d20b2852146c44b4e1241bce9c6f5161b 100644 --- a/flang/test/Fir/array-coor.fir +++ b/flang/test/Fir/array-coor.fir @@ -9,12 +9,12 @@ func.func @array_coor_box_value(%29 : !fir.box>, } // CHECK-LABEL: define double @array_coor_box_value -// CHECK: %[[t3:.*]] = sub i64 %{{.*}}, 1 -// CHECK: %[[t4:.*]] = mul i64 %[[t3]], 1 +// CHECK: %[[t3:.*]] = sub nsw i64 %{{.*}}, 1 +// CHECK: %[[t4:.*]] = mul nsw i64 %[[t3]], 1 // CHECK: %[[t5:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %{{.*}}, i32 0, i32 7, i32 0, i32 2 // CHECK: %[[t6:.*]] = load i64, ptr %[[t5]] -// CHECK: %[[t7:.*]] = mul i64 %[[t4]], %[[t6]] -// CHECK: %[[t8:.*]] = add i64 %[[t7]], 0 +// CHECK: %[[t7:.*]] = mul nsw i64 %[[t4]], %[[t6]] +// CHECK: %[[t8:.*]] = add nsw i64 %[[t7]], 0 // CHECK: %[[t9:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %{{.*}}, i32 0, i32 0 // CHECK: %[[t10:.*]] = load ptr, ptr %[[t9]] // CHECK: %[[t11:.*]] = getelementptr i8, ptr %[[t10]], i64 %[[t8]] @@ -36,8 +36,8 @@ func.func private @take_int(%arg0: !fir.ref) -> () // CHECK-SAME: ptr %[[VAL_0:.*]]) // CHECK: %[[VAL_1:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %[[VAL_0]], i32 0, i32 7, i32 0, i32 2 // CHECK: %[[VAL_2:.*]] = load i64, ptr %[[VAL_1]] -// CHECK: %[[VAL_3:.*]] = mul i64 1, %[[VAL_2]] -// CHECK: %[[VAL_4:.*]] = add i64 %[[VAL_3]], 0 +// CHECK: %[[VAL_3:.*]] = mul nsw i64 1, %[[VAL_2]] +// CHECK: %[[VAL_4:.*]] = add nsw i64 %[[VAL_3]], 0 // CHECK: %[[VAL_5:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %[[VAL_0]], i32 0, i32 0 // CHECK: %[[VAL_6:.*]] = load ptr, ptr %[[VAL_5]] // CHECK: %[[VAL_7:.*]] = getelementptr i8, ptr %[[VAL_6]], i64 %[[VAL_4]] diff --git a/flang/test/Fir/arrexp.fir b/flang/test/Fir/arrexp.fir index 87a276360825057926a92b725bcafe0360ef9224..5d265a5e3a08d678a62d6f93b16f09c6d1c592fd 100644 --- a/flang/test/Fir/arrexp.fir +++ b/flang/test/Fir/arrexp.fir @@ -114,8 +114,8 @@ func.func @f5(%arg0: !fir.box>, %arg1: !fir.box (!fir.array) { // CHECK: %[[B_STRIDE_GEP:.*]] = getelementptr {{.*}}, ptr %[[B]], i32 0, i32 7, i32 0, i32 2 // CHECK: %[[B_STRIDE:.*]] = load i64, ptr %[[B_STRIDE_GEP]] - // CHECK: %[[B_DIM_OFFSET:.*]] = mul i64 %{{.*}}, %[[B_STRIDE]] - // CHECK: %[[B_OFFSET:.*]] = add i64 %[[B_DIM_OFFSET]], 0 + // CHECK: %[[B_DIM_OFFSET:.*]] = mul nsw i64 %{{.*}}, %[[B_STRIDE]] + // CHECK: %[[B_OFFSET:.*]] = add nsw i64 %[[B_DIM_OFFSET]], 0 // CHECK: %[[B_BASE_GEP:.*]] = getelementptr {{.*}}, ptr %{{.*}}, i32 0, i32 0 // CHECK: %[[B_BASE:.*]] = load ptr, ptr %[[B_BASE_GEP]] // CHECK: %[[B_VOID_ADDR:.*]] = getelementptr i8, ptr %[[B_BASE]], i64 %[[B_OFFSET]] @@ -172,7 +172,7 @@ func.func @f7(%arg0: !fir.ref, %arg1: !fir.box>) { %0 = fir.shift %c4 : (index) -> !fir.shift<1> // CHECK: %[[STRIDE_GEP:.*]] = getelementptr {{.*}}, ptr %[[Y]], i32 0, i32 7, i32 0, i32 2 // CHECK: %[[STRIDE:.*]] = load i64, ptr %[[STRIDE_GEP]] - // CHECK: mul i64 96, %[[STRIDE]] + // CHECK: mul nsw i64 96, %[[STRIDE]] %1 = fir.array_coor %arg1(%0) %c100 : (!fir.box>, !fir.shift<1>, index) -> !fir.ref %2 = fir.load %1 : !fir.ref fir.store %2 to %arg0 : !fir.ref @@ -202,7 +202,7 @@ func.func @f8(%a : !fir.ref>>, %i : i32) { func.func @f9(%i: i32, %e : i64, %j: i64, %c: !fir.ref>>) -> !fir.ref> { %s = fir.shape %e, %e : (i64, i64) -> !fir.shape<2> // CHECK: %[[CAST:.*]] = sext i32 %[[I]] to i64 - // CHECK: %[[OFFSET:.*]] = mul i64 %{{.*}}, %[[CAST]] + // CHECK: %[[OFFSET:.*]] = mul nsw i64 %{{.*}}, %[[CAST]] // CHECK: getelementptr i8, ptr %[[C]], i64 %[[OFFSET]] %a = fir.array_coor %c(%s) %j, %j typeparams %i : (!fir.ref>>, !fir.shape<2>, i64, i64, i32) -> !fir.ref> return %a : !fir.ref> diff --git a/flang/test/Fir/convert-to-llvm.fir b/flang/test/Fir/convert-to-llvm.fir index 993058ebb0a4d9e62b3ee3800a313a9c834f099f..be82ffab7e33ef3a6a5324799736adaec19209fb 100644 --- a/flang/test/Fir/convert-to-llvm.fir +++ b/flang/test/Fir/convert-to-llvm.fir @@ -2027,10 +2027,10 @@ func.func @ext_array_coor0(%arg0: !fir.ref>) { // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[C0_1:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C1]] : i64 -// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] : i64 -// CHECK: %[[SC:.*]] = llvm.mul %[[DIFF0]], %[[C1]] : i64 -// CHECK: %[[OFFSET:.*]] = llvm.add %[[SC]], %[[C0_1]] : i64 +// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C1]] overflow : i64 +// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] overflow : i64 +// CHECK: %[[SC:.*]] = llvm.mul %[[DIFF0]], %[[C1]] overflow : i64 +// CHECK: %[[OFFSET:.*]] = llvm.add %[[SC]], %[[C0_1]] overflow : i64 // CHECK: %{{.*}} = llvm.getelementptr %[[ARG0]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32 // Conversion with shift and slice. @@ -2046,12 +2046,12 @@ func.func @ext_array_coor1(%arg0: !fir.ref>) { // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[C0_1:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C0]] : i64 -// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C0]] : i64 -// CHECK: %[[ADJ:.*]] = llvm.sub %[[C0]], %[[C0]] : i64 -// CHECK: %[[DIFF1:.*]] = llvm.add %[[DIFF0]], %[[ADJ]] : i64 -// CHECK: %[[STRIDE:.*]] = llvm.mul %[[DIFF1]], %[[C1]] : i64 -// CHECK: %[[OFFSET:.*]] = llvm.add %[[STRIDE]], %[[C0_1]] : i64 +// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C0]] overflow : i64 +// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C0]] overflow : i64 +// CHECK: %[[ADJ:.*]] = llvm.sub %[[C0]], %[[C0]] overflow : i64 +// CHECK: %[[DIFF1:.*]] = llvm.add %[[DIFF0]], %[[ADJ]] overflow : i64 +// CHECK: %[[STRIDE:.*]] = llvm.mul %[[DIFF1]], %[[C1]] overflow : i64 +// CHECK: %[[OFFSET:.*]] = llvm.add %[[STRIDE]], %[[C0_1]] overflow : i64 // CHECK: %{{.*}} = llvm.getelementptr %[[ARG0]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32 // Conversion for a dynamic length char. @@ -2067,10 +2067,10 @@ func.func @ext_array_coor2(%arg0: !fir.ref>>) { // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[C0_1:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C1]] : i64 -// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] : i64 -// CHECK: %[[SC:.*]] = llvm.mul %[[DIFF0]], %[[C1]] : i64 -// CHECK: %[[OFFSET:.*]] = llvm.add %[[SC]], %[[C0_1]] : i64 +// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C1]] overflow : i64 +// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] overflow : i64 +// CHECK: %[[SC:.*]] = llvm.mul %[[DIFF0]], %[[C1]] overflow : i64 +// CHECK: %[[OFFSET:.*]] = llvm.add %[[SC]], %[[C0_1]] overflow : i64 // CHECK: %{{.*}} = llvm.getelementptr %[[ARG0]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 // Conversion for a `fir.box`. @@ -2086,12 +2086,12 @@ func.func @ext_array_coor3(%arg0: !fir.box>) { // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[C0_1:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C1]] : i64 -// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] : i64 +// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C1]] overflow : i64 +// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] overflow : i64 // CHECK: %[[GEPSTRIDE:.*]] = llvm.getelementptr %[[ARG0]][0, 7, 0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)> // CHECK: %[[LOADEDSTRIDE:.*]] = llvm.load %[[GEPSTRIDE]] : !llvm.ptr -> i64 -// CHECK: %[[SC:.*]] = llvm.mul %[[DIFF0]], %[[LOADEDSTRIDE]] : i64 -// CHECK: %[[OFFSET:.*]] = llvm.add %[[SC]], %[[C0_1]] : i64 +// CHECK: %[[SC:.*]] = llvm.mul %[[DIFF0]], %[[LOADEDSTRIDE]] overflow : i64 +// CHECK: %[[OFFSET:.*]] = llvm.add %[[SC]], %[[C0_1]] overflow : i64 // CHECK: %[[GEPADDR:.*]] = llvm.getelementptr %[[ARG0]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)> // CHECK: %[[LOADEDADDR:.*]] = llvm.load %[[GEPADDR]] : !llvm.ptr -> !llvm.ptr // CHECK: %[[GEPADDROFFSET:.*]] = llvm.getelementptr %[[LOADEDADDR]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 @@ -2115,12 +2115,12 @@ func.func @ext_array_coor4(%arg0: !fir.ref>) { // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[C1_1:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[C0_1:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[IDX:.*]] = llvm.sub %[[C1]], %[[C0]] : i64 -// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] : i64 -// CHECK: %[[ADJ:.*]] = llvm.sub %[[C10]], %[[C0]] : i64 -// CHECK: %[[DIFF1:.*]] = llvm.add %[[DIFF0]], %[[ADJ]] : i64 -// CHECK: %[[STRIDE:.*]] = llvm.mul %[[DIFF1]], %[[C1_1]] : i64 -// CHECK: %[[OFFSET:.*]] = llvm.add %[[STRIDE]], %[[C0_1]] : i64 +// CHECK: %[[IDX:.*]] = llvm.sub %[[C1]], %[[C0]] overflow : i64 +// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] overflow : i64 +// CHECK: %[[ADJ:.*]] = llvm.sub %[[C10]], %[[C0]] overflow : i64 +// CHECK: %[[DIFF1:.*]] = llvm.add %[[DIFF0]], %[[ADJ]] overflow : i64 +// CHECK: %[[STRIDE:.*]] = llvm.mul %[[DIFF1]], %[[C1_1]] overflow : i64 +// CHECK: %[[OFFSET:.*]] = llvm.add %[[STRIDE]], %[[C0_1]] overflow : i64 // CHECK: %{{.*}} = llvm.getelementptr %[[ARG0]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32 // Conversion with index type shape and slice @@ -2134,13 +2134,13 @@ func.func @ext_array_coor5(%arg0: !fir.ref>, %idx1 : index, %i // CHECK-SAME: %[[VAL_0:.*]]: !llvm.ptr, %[[VAL_1:.*]]: i64, %[[VAL_2:.*]]: i64, %[[VAL_3:.*]]: i64, %[[VAL_4:.*]]: i64, %[[VAL_5:.*]]: i64) { // CHECK: %[[VAL_6:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[VAL_7:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[VAL_8:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_9:.*]] = llvm.mul %[[VAL_8]], %[[VAL_4]] : i64 -// CHECK: %[[VAL_10:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_11:.*]] = llvm.add %[[VAL_9]], %[[VAL_10]] : i64 -// CHECK: %[[VAL_12:.*]] = llvm.mul %[[VAL_11]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_13:.*]] = llvm.add %[[VAL_12]], %[[VAL_7]] : i64 -// CHECK: %[[VAL_14:.*]] = llvm.mul %[[VAL_6]], %[[VAL_1]] : i64 +// CHECK: %[[VAL_8:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_9:.*]] = llvm.mul %[[VAL_8]], %[[VAL_4]] overflow : i64 +// CHECK: %[[VAL_10:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_11:.*]] = llvm.add %[[VAL_9]], %[[VAL_10]] overflow : i64 +// CHECK: %[[VAL_12:.*]] = llvm.mul %[[VAL_11]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_13:.*]] = llvm.add %[[VAL_12]], %[[VAL_7]] overflow : i64 +// CHECK: %[[VAL_14:.*]] = llvm.mul %[[VAL_6]], %[[VAL_1]] overflow : i64 // CHECK: %[[VAL_16:.*]] = llvm.getelementptr %[[VAL_0]][%[[VAL_13]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32 // CHECK: } @@ -2155,27 +2155,27 @@ func.func @ext_array_coor6(%arg0: !fir.ref>, %idx1 : index // CHECK-SAME: %[[VAL_0:.*]]: !llvm.ptr, %[[VAL_1:.*]]: i64, %[[VAL_2:.*]]: i64, %[[VAL_3:.*]]: i64, %[[VAL_4:.*]]: i64, %[[VAL_5:.*]]: i64) { // CHECK: %[[VAL_6:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[VAL_7:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[VAL_8:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_9:.*]] = llvm.mul %[[VAL_8]], %[[VAL_4]] : i64 -// CHECK: %[[VAL_10:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_11:.*]] = llvm.add %[[VAL_9]], %[[VAL_10]] : i64 -// CHECK: %[[VAL_12:.*]] = llvm.mul %[[VAL_11]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_13:.*]] = llvm.add %[[VAL_12]], %[[VAL_7]] : i64 -// CHECK: %[[VAL_14:.*]] = llvm.mul %[[VAL_6]], %[[VAL_1]] : i64 -// CHECK: %[[VAL_15:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_16:.*]] = llvm.mul %[[VAL_15]], %[[VAL_4]] : i64 -// CHECK: %[[VAL_17:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_18:.*]] = llvm.add %[[VAL_16]], %[[VAL_17]] : i64 -// CHECK: %[[VAL_19:.*]] = llvm.mul %[[VAL_18]], %[[VAL_14]] : i64 -// CHECK: %[[VAL_20:.*]] = llvm.add %[[VAL_19]], %[[VAL_13]] : i64 -// CHECK: %[[VAL_21:.*]] = llvm.mul %[[VAL_14]], %[[VAL_1]] : i64 -// CHECK: %[[VAL_22:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_23:.*]] = llvm.mul %[[VAL_22]], %[[VAL_4]] : i64 -// CHECK: %[[VAL_24:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_25:.*]] = llvm.add %[[VAL_23]], %[[VAL_24]] : i64 -// CHECK: %[[VAL_26:.*]] = llvm.mul %[[VAL_25]], %[[VAL_21]] : i64 -// CHECK: %[[VAL_27:.*]] = llvm.add %[[VAL_26]], %[[VAL_20]] : i64 -// CHECK: %[[VAL_28:.*]] = llvm.mul %[[VAL_21]], %[[VAL_1]] : i64 +// CHECK: %[[VAL_8:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_9:.*]] = llvm.mul %[[VAL_8]], %[[VAL_4]] overflow : i64 +// CHECK: %[[VAL_10:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_11:.*]] = llvm.add %[[VAL_9]], %[[VAL_10]] overflow : i64 +// CHECK: %[[VAL_12:.*]] = llvm.mul %[[VAL_11]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_13:.*]] = llvm.add %[[VAL_12]], %[[VAL_7]] overflow : i64 +// CHECK: %[[VAL_14:.*]] = llvm.mul %[[VAL_6]], %[[VAL_1]] overflow : i64 +// CHECK: %[[VAL_15:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_16:.*]] = llvm.mul %[[VAL_15]], %[[VAL_4]] overflow : i64 +// CHECK: %[[VAL_17:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_18:.*]] = llvm.add %[[VAL_16]], %[[VAL_17]] overflow : i64 +// CHECK: %[[VAL_19:.*]] = llvm.mul %[[VAL_18]], %[[VAL_14]] overflow : i64 +// CHECK: %[[VAL_20:.*]] = llvm.add %[[VAL_19]], %[[VAL_13]] overflow : i64 +// CHECK: %[[VAL_21:.*]] = llvm.mul %[[VAL_14]], %[[VAL_1]] overflow : i64 +// CHECK: %[[VAL_22:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_23:.*]] = llvm.mul %[[VAL_22]], %[[VAL_4]] overflow : i64 +// CHECK: %[[VAL_24:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_25:.*]] = llvm.add %[[VAL_23]], %[[VAL_24]] overflow : i64 +// CHECK: %[[VAL_26:.*]] = llvm.mul %[[VAL_25]], %[[VAL_21]] overflow : i64 +// CHECK: %[[VAL_27:.*]] = llvm.add %[[VAL_26]], %[[VAL_20]] overflow : i64 +// CHECK: %[[VAL_28:.*]] = llvm.mul %[[VAL_21]], %[[VAL_1]] overflow : i64 // CHECK: %[[VAL_30:.*]] = llvm.getelementptr %[[VAL_0]][%[[VAL_27]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32 // CHECK: llvm.return // CHECK: } @@ -2193,13 +2193,13 @@ func.func @ext_array_coor_dt_slice(%arg0: !fir.ref : i64 +// CHECK: %[[VAL_10:.*]] = llvm.mul %[[VAL_9]], %[[VAL_4]] overflow : i64 +// CHECK: %[[VAL_11:.*]] = llvm.sub %[[VAL_2]], %[[VAL_7]] overflow : i64 +// CHECK: %[[VAL_12:.*]] = llvm.add %[[VAL_10]], %[[VAL_11]] overflow : i64 +// CHECK: %[[VAL_13:.*]] = llvm.mul %[[VAL_12]], %[[VAL_7]] overflow : i64 +// CHECK: %[[VAL_14:.*]] = llvm.add %[[VAL_13]], %[[VAL_8]] overflow : i64 +// CHECK: %[[VAL_15:.*]] = llvm.mul %[[VAL_7]], %[[VAL_1]] overflow : i64 // CHECK: %[[VAL_17:.*]] = llvm.getelementptr %[[VAL_0]][%[[VAL_14]], 0] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<"_QFtest_dt_sliceTt", (i32, i32)> // CHECK: llvm.return // CHECK: } @@ -2452,8 +2452,8 @@ func.func @coordinate_box_array_1d(%arg0: !fir.box>, %arg1: // Index of the 1st CFI_dim_t object (corresonds the the 1st dimension) // CHECK-NEXT: %[[DIM_1_MEM_STRIDE_ADDR:.*]] = llvm.getelementptr %[[BOX]][0, 7, 0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)> // CHECK-NEXT: %[[DIM_1_MEM_STRIDE_VAL:.*]] = llvm.load %[[DIM_1_MEM_STRIDE_ADDR]] : !llvm.ptr -> i64 -// CHECK-NEXT: %[[BYTE_OFFSET:.*]] = llvm.mul %[[COORDINATE]], %[[DIM_1_MEM_STRIDE_VAL]] : i64 -// CHECK-NEXT: %[[SUBOJECT_OFFSET:.*]] = llvm.add %[[BYTE_OFFSET]], %[[OFFSET_INIT]] : i64 +// CHECK-NEXT: %[[BYTE_OFFSET:.*]] = llvm.mul %[[COORDINATE]], %[[DIM_1_MEM_STRIDE_VAL]] overflow : i64 +// CHECK-NEXT: %[[SUBOJECT_OFFSET:.*]] = llvm.add %[[BYTE_OFFSET]], %[[OFFSET_INIT]] overflow : i64 // CHECK-NEXT: %[[SUBOBJECT_ADDR:.*]] = llvm.getelementptr %[[ARRAY_OBJECT]][%[[SUBOJECT_OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 // CHECK-NEXT: llvm.return @@ -2471,8 +2471,8 @@ func.func @coordinate_of_box_dynamic_array_1d(%arg0: !fir.box !llvm.ptr, !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)> // CHECK-NEXT: %[[DIM_1_MEM_STRIDE_VAL:.*]] = llvm.load %[[DIM_1_MEM_STRIDE_ADDR]] : !llvm.ptr -> i64 -// CHECK-NEXT: %[[BYTE_OFFSET:.*]] = llvm.mul %[[COORDINATE]], %[[DIM_1_MEM_STRIDE_VAL]] : i64 -// CHECK-NEXT: %[[SUBOJECT_OFFSET:.*]] = llvm.add %[[BYTE_OFFSET]], %[[OFFSET_INIT]] : i64 +// CHECK-NEXT: %[[BYTE_OFFSET:.*]] = llvm.mul %[[COORDINATE]], %[[DIM_1_MEM_STRIDE_VAL]] overflow : i64 +// CHECK-NEXT: %[[SUBOJECT_OFFSET:.*]] = llvm.add %[[BYTE_OFFSET]], %[[OFFSET_INIT]] overflow : i64 // CHECK-NEXT: %[[SUBOBJECT_ADDR:.*]] = llvm.getelementptr %[[ARRAY_OBJECT]][%[[SUBOJECT_OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 // CHECK-NEXT: llvm.return @@ -2492,13 +2492,13 @@ func.func @coordinate_box_array_2d(%arg0: !fir.box>, % // Index of the 1st CFI_dim_t object (corresonds the the 1st dimension) // CHECK-NEXT: %[[DIM_1_MEM_STRIDE_ADDR:.*]] = llvm.getelementptr %[[BOX]][0, 7, 0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<2 x array<3 x i64>>)> // CHECK-NEXT: %[[DIM_1_MEM_STRIDE_VAL:.*]] = llvm.load %[[DIM_1_MEM_STRIDE_ADDR]] : !llvm.ptr -> i64 -// CHECK-NEXT: %[[BYTE_OFFSET_1:.*]] = llvm.mul %[[COORDINATE_1]], %[[DIM_1_MEM_STRIDE_VAL]] : i64 -// CHECK-NEXT: %[[SUBOBJECT_OFFSET_1:.*]] = llvm.add %[[BYTE_OFFSET]], %[[OFFSET_INIT]] : i64 +// CHECK-NEXT: %[[BYTE_OFFSET_1:.*]] = llvm.mul %[[COORDINATE_1]], %[[DIM_1_MEM_STRIDE_VAL]] overflow : i64 +// CHECK-NEXT: %[[SUBOBJECT_OFFSET_1:.*]] = llvm.add %[[BYTE_OFFSET]], %[[OFFSET_INIT]] overflow : i64 // Index of the 1st CFI_dim_t object (corresonds the the 2nd dimension) // CHECK-NEXT: %[[DIM_2_MEM_STRIDE_ADDR:.*]] = llvm.getelementptr %[[BOX]][0, 7, 1, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<2 x array<3 x i64>>)> // CHECK-NEXT: %[[DIM_2_MEM_STRIDE_VAL:.*]] = llvm.load %[[DIM_2_MEM_STRIDE_ADDR]] : !llvm.ptr -> i64 -// CHECK-NEXT: %[[BYTE_OFFSET_2:.*]] = llvm.mul %[[COORDINATE_2]], %[[DIM_2_MEM_STRIDE_VAL]] : i64 -// CHECK-NEXT: %[[SUBOBJECT_OFFSET_2:.*]] = llvm.add %[[BYTE_OFFSET_2]], %[[SUBOBJECT_OFFSET_1]] : i64 +// CHECK-NEXT: %[[BYTE_OFFSET_2:.*]] = llvm.mul %[[COORDINATE_2]], %[[DIM_2_MEM_STRIDE_VAL]] overflow : i64 +// CHECK-NEXT: %[[SUBOBJECT_OFFSET_2:.*]] = llvm.add %[[BYTE_OFFSET_2]], %[[SUBOBJECT_OFFSET_1]] overflow : i64 // CHECK-NEXT: %[[SUBOBJECT_ADDR:.*]] = llvm.getelementptr %[[ARRAY_OBJECT]][%[[SUBOBJECT_OFFSET_2]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 // CHECK-NEXT: llvm.return @@ -2520,8 +2520,8 @@ func.func @coordinate_box_derived_inside_array(%arg0: !fir.box !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)> // CHECK: %[[VAL_14:.*]] = llvm.load %[[VAL_13]] : !llvm.ptr -> i64 -// CHECK: %[[VAL_15:.*]] = llvm.mul %[[COORDINATE_1]], %[[VAL_14]] : i64 -// CHECK: %[[OFFSET:.*]] = llvm.add %[[VAL_15]], %[[VAL_8]] : i64 +// CHECK: %[[VAL_15:.*]] = llvm.mul %[[COORDINATE_1]], %[[VAL_14]] overflow : i64 +// CHECK: %[[OFFSET:.*]] = llvm.add %[[VAL_15]], %[[VAL_8]] overflow : i64 // CHECK: %[[DERIVED:.*]] = llvm.getelementptr %[[ARRAY]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 // CHECK: %[[VAL_20:.*]] = llvm.getelementptr %[[DERIVED]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"derived_3", (f32, f32)> // CHECK: llvm.return diff --git a/flang/test/Fir/coordinateof.fir b/flang/test/Fir/coordinateof.fir index c605f43a154373fbcc538d7a6421d1a73e7144e7..acb9fd57c09556818bd3e7b96be5b9a96b750f3e 100644 --- a/flang/test/Fir/coordinateof.fir +++ b/flang/test/Fir/coordinateof.fir @@ -30,8 +30,8 @@ func.func @foo3(%box : !fir.box>, %i : i32) -> i32 { // CHECK: %[[boxptr:.*]] = load ptr, ptr %[[gep0]] // CHECK: %[[gep1:.*]] = getelementptr { ptr, i64, {{.*}} i32 7 // CHECK: %[[stride:.*]] = load i64, ptr %[[gep1]] - // CHECK: %[[dimoffset:.*]] = mul i64 %[[cvt]], %[[stride]] - // CHECK: %[[offset:.*]] = add i64 %[[dimoffset]], 0 + // CHECK: %[[dimoffset:.*]] = mul nsw i64 %[[cvt]], %[[stride]] + // CHECK: %[[offset:.*]] = add nsw i64 %[[dimoffset]], 0 // CHECK: %[[gep2:.*]] = getelementptr i8, ptr %[[boxptr]], i64 %[[offset]] %1 = fir.coordinate_of %box, %ii : (!fir.box>, index) -> !fir.ref // CHECK: load i32, ptr %[[gep2]] @@ -68,8 +68,8 @@ func.func @foo6(%box : !fir.box>>>, %i : i64 // CHECK: %[[addr:.*]] = load ptr, ptr %[[addr_gep]] // CHECK: %[[stride_gep:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[box]], i32 0, i32 7, i32 0, i32 2 // CHECK: %[[stride:.*]] = load i64, ptr %[[stride_gep]] - // CHECK: %[[mul:.*]] = mul i64 %{{.*}}, %[[stride]] - // CHECK: %[[offset:.*]] = add i64 %[[mul]], 0 + // CHECK: %[[mul:.*]] = mul nsw i64 %{{.*}}, %[[stride]] + // CHECK: %[[offset:.*]] = add nsw i64 %[[mul]], 0 // CHECK: %[[gep:.*]] = getelementptr i8, ptr %[[addr]], i64 %[[offset]] %coor = fir.coordinate_of %box, %i : (!fir.box>>>, i64) -> !fir.ref> diff --git a/flang/test/Fir/tbaa.fir b/flang/test/Fir/tbaa.fir index bbc8a81bdf6839b43f410aa7d9b592679207933a..4474bbbe3dd7473505b8a287f4c640636878f969 100644 --- a/flang/test/Fir/tbaa.fir +++ b/flang/test/Fir/tbaa.fir @@ -41,8 +41,8 @@ module { // CHECK: %[[VAL_8:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_9:.*]] = llvm.getelementptr %[[VAL_0]][0, 7, 0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)> // CHECK: %[[VAL_10:.*]] = llvm.load %[[VAL_9]] {tbaa = [#[[$BOXT]]]} : !llvm.ptr -> i64 -// CHECK: %[[VAL_11:.*]] = llvm.mul %[[VAL_4]], %[[VAL_10]] : i64 -// CHECK: %[[VAL_12:.*]] = llvm.add %[[VAL_11]], %[[VAL_8]] : i64 +// CHECK: %[[VAL_11:.*]] = llvm.mul %[[VAL_4]], %[[VAL_10]] overflow : i64 +// CHECK: %[[VAL_12:.*]] = llvm.add %[[VAL_11]], %[[VAL_8]] overflow : i64 // CHECK: %[[VAL_14:.*]] = llvm.getelementptr %[[VAL_7]]{{\[}}%[[VAL_12]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 // CHECK: %[[VAL_16:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_17:.*]] = llvm.mlir.constant(-1 : i32) : i32 @@ -330,12 +330,12 @@ func.func @tbaa(%arg0: !fir.box>) { // CHECK: %[[VAL_1:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_2:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[VAL_3:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[VAL_4:.*]] = llvm.sub %[[VAL_1]], %[[VAL_2]] : i64 -// CHECK: %[[VAL_5:.*]] = llvm.mul %[[VAL_4]], %[[VAL_2]] : i64 +// CHECK: %[[VAL_4:.*]] = llvm.sub %[[VAL_1]], %[[VAL_2]] overflow : i64 +// CHECK: %[[VAL_5:.*]] = llvm.mul %[[VAL_4]], %[[VAL_2]] overflow : i64 // CHECK: %[[VAL_6:.*]] = llvm.getelementptr %[[VAL_0]][0, 7, 0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> // CHECK: %[[VAL_7:.*]] = llvm.load %[[VAL_6]] {tbaa = [#[[$BOXT]]]} : !llvm.ptr -> i64 -// CHECK: %[[VAL_8:.*]] = llvm.mul %[[VAL_5]], %[[VAL_7]] : i64 -// CHECK: %[[VAL_9:.*]] = llvm.add %[[VAL_8]], %[[VAL_3]] : i64 +// CHECK: %[[VAL_8:.*]] = llvm.mul %[[VAL_5]], %[[VAL_7]] overflow : i64 +// CHECK: %[[VAL_9:.*]] = llvm.add %[[VAL_8]], %[[VAL_3]] overflow : i64 // CHECK: %[[VAL_10:.*]] = llvm.getelementptr %[[VAL_0]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> // CHECK: %[[VAL_11:.*]] = llvm.load %[[VAL_10]] {tbaa = [#[[$BOXT]]]} : !llvm.ptr -> !llvm.ptr // CHECK: %[[VAL_13:.*]] = llvm.getelementptr %[[VAL_11]]{{\[}}%[[VAL_9]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 diff --git a/flang/test/Lower/HLFIR/implicit-type-conversion-allocatable.f90 b/flang/test/Lower/HLFIR/implicit-type-conversion-allocatable.f90 index 7083a825dfd3b429ed51604c7eab8f77f1e1ea6b..361cd61adea2dd96c0d864f2efe2b45f4eb76598 100644 --- a/flang/test/Lower/HLFIR/implicit-type-conversion-allocatable.f90 +++ b/flang/test/Lower/HLFIR/implicit-type-conversion-allocatable.f90 @@ -38,3 +38,29 @@ end subroutine ! CHECK: hlfir.destroy %[[VAL_8]] : !hlfir.expr ! CHECK: return ! CHECK: } + +! Test that RHS character length is preserved in a character KIND +! conversion before the assignment. +subroutine kind_and_length(a, b) + character(len=4,kind=4), allocatable :: a(:) + character(len=2,kind=1) :: b(:) + a = b +end subroutine +! CHECK-LABEL: func.func @_QPkind_and_length( +! CHECK: %[[VAL_2:.*]] = arith.constant 4 : index +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] typeparams %[[VAL_2:[a-z0-9]*]] {{.*}}Ea +! CHECK: %[[VAL_4:.*]] = arith.constant 2 : index +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]] typeparams %[[VAL_4:[a-z0-9]*]] {{.*}}Eb +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : index +! CHECK: %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_5]]#0, %[[VAL_6]] : (!fir.box>>, index) -> (index, index, index) +! CHECK: %[[VAL_8:.*]] = fir.shape %[[VAL_7]]#1 : (index) -> !fir.shape<1> +! CHECK: %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] typeparams %[[VAL_4]] unordered : (!fir.shape<1>, index) -> !hlfir.expr> { +! CHECK: ^bb0(%[[VAL_10:.*]]: index): +! CHECK: %[[VAL_11:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_10]]) typeparams %[[VAL_4]] : (!fir.box>>, index, index) -> !fir.ref> +! CHECK: %[[VAL_12:.*]] = fir.alloca !fir.char<4,?>(%[[VAL_4]] : index) +! CHECK: fir.char_convert %[[VAL_11]] for %[[VAL_4]] to %[[VAL_12]] : !fir.ref>, index, !fir.ref> +! CHECK: %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12:[a-z0-9]*]] typeparams %[[VAL_4:[a-z0-9]*]] {uniq_name = ".temp.kindconvert"} : (!fir.ref>, index) -> (!fir.boxchar<4>, !fir.ref>) +! CHECK: hlfir.yield_element %[[VAL_13]]#0 : !fir.boxchar<4> +! CHECK: } +! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_3]]#0 realloc keep_lhs_len : !hlfir.expr>, !fir.ref>>>> + diff --git a/flang/test/Lower/OpenACC/acc-set.f90 b/flang/test/Lower/OpenACC/acc-set.f90 index 2741c3c6325590641a4b909611b81772901f8016..31c1912a87f57e010328e12bc4ce95dccc5dc5a2 100644 --- a/flang/test/Lower/OpenACC/acc-set.f90 +++ b/flang/test/Lower/OpenACC/acc-set.f90 @@ -33,7 +33,7 @@ end ! CHECK: %[[C0:.*]] = arith.constant 0 : i32 ! CHECK: acc.set device_num(%[[C0]] : i32) -! CHECK: acc.set attributes {device_type = #acc.device_type<*>} +! CHECK: acc.set attributes {device_type = #acc.device_type} ! CHECK: acc.set attributes {device_type = #acc.device_type} diff --git a/flang/test/Lower/OpenACC/acc-update.f90 b/flang/test/Lower/OpenACC/acc-update.f90 index 2e8d672a43eb4151676fd9fa3f879fb367eed77d..d2b15f8bd258e7aeaa2dc6fbbe497fade00cbecc 100644 --- a/flang/test/Lower/OpenACC/acc-update.f90 +++ b/flang/test/Lower/OpenACC/acc-update.f90 @@ -114,7 +114,7 @@ subroutine acc_update !$acc update host(a) device_type(*) ! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {dataClause = #acc, name = "a", structured = false} -! CHECK: acc.update dataOperands(%[[DEVPTR_A]] : !fir.ref>) attributes {device_types = [#acc.device_type<*>]} +! CHECK: acc.update dataOperands(%[[DEVPTR_A]] : !fir.ref>) attributes {device_types = [#acc.device_type]} ! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#1 : !fir.ref>) {name = "a", structured = false} end subroutine acc_update diff --git a/flang/test/Lower/PowerPC/ppc-vec-store-elem-order.f90 b/flang/test/Lower/PowerPC/ppc-vec-store-elem-order.f90 index 494ed21f4fe928be92707e4f6e470c65a6856454..caf6d5463a833632b8117b84030e2b769b84385b 100644 --- a/flang/test/Lower/PowerPC/ppc-vec-store-elem-order.f90 +++ b/flang/test/Lower/PowerPC/ppc-vec-store-elem-order.f90 @@ -67,10 +67,10 @@ subroutine vec_xstd2_test(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x float>, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x float>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i16, ptr %1, align 2 @@ -93,10 +93,10 @@ subroutine vec_xstw4_test(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x float>, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x float>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i16, ptr %1, align 2 diff --git a/flang/test/Lower/PowerPC/ppc-vec-store.f90 b/flang/test/Lower/PowerPC/ppc-vec-store.f90 index 8e20228d68259890c5339184159006aeacb37b9e..c25cc8b07cf7991e012adbbf17361feb34b984e8 100644 --- a/flang/test/Lower/PowerPC/ppc-vec-store.f90 +++ b/flang/test/Lower/PowerPC/ppc-vec-store.f90 @@ -89,10 +89,10 @@ subroutine vec_st_vi4i4via4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[iextsub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[iextmul:.*]] = mul i64 %[[iextsub]], 1 -! LLVMIR: %[[iextmul2:.*]] = mul i64 %[[iextmul]], 1 -! LLVMIR: %[[iextadd:.*]] = add i64 %[[iextmul2]], 0 +! LLVMIR: %[[iextsub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[iextmul:.*]] = mul nsw i64 %[[iextsub]], 1 +! LLVMIR: %[[iextmul2:.*]] = mul nsw i64 %[[iextmul]], 1 +! LLVMIR: %[[iextadd:.*]] = add nsw i64 %[[iextmul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x i32>, ptr %2, i64 %[[iextadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i32, ptr %1, align 4 @@ -206,10 +206,10 @@ subroutine vec_ste_vi4i4ia4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr i32, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i32, ptr %1, align 4 @@ -244,10 +244,10 @@ subroutine vec_stxv_test_vi4i8ia4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr i32, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i64, ptr %1, align 8 @@ -278,10 +278,10 @@ subroutine vec_stxv_test_vi4i4vai4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x i32>, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i32, ptr %1, align 4 @@ -317,10 +317,10 @@ subroutine vec_xst_test_vi4i8ia4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr i32, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i64, ptr %1, align 8 @@ -351,10 +351,10 @@ subroutine vec_xst_test_vi4i4vai4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x i32>, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i32, ptr %1, align 4 @@ -390,10 +390,10 @@ subroutine vec_xst_be_test_vi4i8ia4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr i32, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i64, ptr %1, align 8 @@ -426,10 +426,10 @@ subroutine vec_xst_be_test_vi4i4vai4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x i32>, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i32, ptr %1, align 4 @@ -467,10 +467,10 @@ subroutine vec_xstd2_test_vi4i8ia4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr i32, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i64, ptr %1, align 8 @@ -503,10 +503,10 @@ subroutine vec_xstd2_test_vi4i4vai4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x i32>, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i32, ptr %1, align 4 @@ -543,10 +543,10 @@ subroutine vec_xstw4_test_vi4i8ia4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr i32, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i64, ptr %1, align 8 @@ -578,10 +578,10 @@ subroutine vec_xstw4_test_vi4i4vai4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x i32>, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i32, ptr %1, align 4 diff --git a/flang/test/Lower/RISCV/riscv-vector-bits-vscale-range.f90 b/flang/test/Lower/RISCV/riscv-vector-bits-vscale-range.f90 new file mode 100644 index 0000000000000000000000000000000000000000..b08ad91feb38844d6fef19cb171e4246e563b90e --- /dev/null +++ b/flang/test/Lower/RISCV/riscv-vector-bits-vscale-range.f90 @@ -0,0 +1,21 @@ +! REQUIRES: riscv-registered-target +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | FileCheck %s -D#VBITS=1 +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=2 -mvscale-max=2 -emit-llvm -o - %s | FileCheck %s -D#VBITS=2 +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=4 -mvscale-max=4 -emit-llvm -o - %s | FileCheck %s -D#VBITS=4 +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=8 -mvscale-max=8 -emit-llvm -o - %s | FileCheck %s -D#VBITS=8 +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=16 -mvscale-max=16 -emit-llvm -o - %s | FileCheck %s -D#VBITS=16 +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=1 -emit-llvm -o - %s | FileCheck %s -D#VBITS=1 --check-prefix=CHECK-NOMAX +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=2 -emit-llvm -o - %s | FileCheck %s -D#VBITS=2 --check-prefix=CHECK-NOMAX +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=4 -emit-llvm -o - %s | FileCheck %s -D#VBITS=4 --check-prefix=CHECK-NOMAX +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=8 -emit-llvm -o - %s | FileCheck %s -D#VBITS=8 --check-prefix=CHECK-NOMAX +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=16 -emit-llvm -o - %s | FileCheck %s -D#VBITS=16 --check-prefix=CHECK-NOMAX +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=1 -mvscale-max=0 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-UNBOUNDED +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-NONE + +! CHECK-LABEL: @func_() #0 +! CHECK: attributes #0 = {{{.*}} vscale_range([[#VBITS]],[[#VBITS]]) {{.*}}} +! CHECK-NOMAX: attributes #0 = {{{.*}} vscale_range([[#VBITS]],0) {{.*}}} +! CHECK-UNBOUNDED: attributes #0 = {{{.*}} vscale_range(1,0) {{.*}}} +! CHECK-NONE-NOT: vscale_range +subroutine func +end subroutine func diff --git a/flang/test/Parser/continuation-before-quote.f90 b/flang/test/Parser/continuation-before-quote.f90 deleted file mode 100644 index 66252010d89c465fd9b5f8ac6f0b6f20a46a8028..0000000000000000000000000000000000000000 --- a/flang/test/Parser/continuation-before-quote.f90 +++ /dev/null @@ -1,10 +0,0 @@ -! RUN: %flang_fc1 -fsyntax-only -pedantic %s 2>&1 | FileCheck %s -! Continuation between repeated quotation marks -subroutine test -!CHECK: portability: Repeated quote mark in character literal continuation line should have been preceded by '&' - print *, 'needs an '& -'ampersand''' -!CHECK-NOT: portability: Repeated quote mark in character literal continuation line should have been preceded by '&' - print *, 'has an '& -&'ampersand''' -end diff --git a/flang/test/Parser/continuation-without-ampersand.f90 b/flang/test/Parser/continuation-without-ampersand.f90 new file mode 100644 index 0000000000000000000000000000000000000000..5c3f23235edccbfe2aeb73aaea82c99c064add4a --- /dev/null +++ b/flang/test/Parser/continuation-without-ampersand.f90 @@ -0,0 +1,13 @@ +! RUN: %flang_fc1 -fsyntax-only -pedantic %s 2>&1 | FileCheck %s +! Continuation between repeated quotation marks +subroutine test +!CHECK: portability: Character literal continuation line should have been preceded by '&' + print *, 'needs an '& +'ampersand''' +!CHECK: portability: Character literal continuation line should have been preceded by '&' + print *, 'also needs an '& + 'ampersand''' +!CHECK-NOT: portability: Character literal continuation line should have been preceded by '&' + print *, 'has an '& +&'ampersand''' +end diff --git a/flang/test/Semantics/OpenMP/target01.f90 b/flang/test/Semantics/OpenMP/target01.f90 index 485fa1f2530c3b7e1fd9d498b2f9b035c951b44c..9836f0112738fe3f3cba4bb224f67380b8aa5f70 100644 --- a/flang/test/Semantics/OpenMP/target01.f90 +++ b/flang/test/Semantics/OpenMP/target01.f90 @@ -39,19 +39,19 @@ subroutine bar(b1, b2, b3) type(c_ptr), pointer :: b2 type(c_ptr), value :: b3 - !ERROR: Variable 'c' in IS_DEVICE_PTR clause must be a dummy argument + !WARNING: Variable 'c' in IS_DEVICE_PTR clause must be a dummy argument. This semantic check is deprecated from OpenMP 5.2 and later. !$omp target is_device_ptr(c) y = y + 1 !$omp end target - !ERROR: Variable 'b1' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. + !WARNING: Variable 'b1' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later. !$omp target is_device_ptr(b1) y = y + 1 !$omp end target - !ERROR: Variable 'b2' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. + !WARNING: Variable 'b2' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later. !$omp target is_device_ptr(b2) y = y + 1 !$omp end target - !ERROR: Variable 'b3' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. + !WARNING: Variable 'b3' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later. !$omp target is_device_ptr(b3) y = y + 1 !$omp end target diff --git a/flang/test/Semantics/definable01.f90 b/flang/test/Semantics/definable01.f90 index fff493fe7a4152fd3c52dc7157877ace57418673..c0f10668fb480eacda91530dad28ba7005c2f427 100644 --- a/flang/test/Semantics/definable01.f90 +++ b/flang/test/Semantics/definable01.f90 @@ -82,4 +82,19 @@ module m subroutine test3b(pp) procedure(sin), pointer, intent(in out) :: pp end subroutine + subroutine test4(p) + type(ptype), pointer, intent(in) :: p + p%x = 1. + p%ptr = 1. ! ok + nullify(p%ptr) ! ok + !CHECK: error: 'p' may not appear in NULLIFY + !CHECK: because: 'p' is an INTENT(IN) dummy argument + nullify(p) + end + subroutine test5(np) + type(ptype), intent(in) :: np + !CHECK: error: 'ptr' may not appear in NULLIFY + !CHECK: because: 'np' is an INTENT(IN) dummy argument + nullify(np%ptr) + end end module diff --git a/flang/unittests/Runtime/NumericalFormatTest.cpp b/flang/unittests/Runtime/NumericalFormatTest.cpp index 219947fe4fbbb794aa1242fabd56b07887449414..b5b8eb05943732cb5b922c3352d302d98b593884 100644 --- a/flang/unittests/Runtime/NumericalFormatTest.cpp +++ b/flang/unittests/Runtime/NumericalFormatTest.cpp @@ -710,8 +710,12 @@ TEST(IOApiTests, FormatDoubleValues) { {"(F5.3,';')", 0.099999, "0.100;"}, {"(F5.3,';')", 0.0099999, "0.010;"}, {"(F5.3,';')", 0.00099999, "0.001;"}, - {"(F5.3,';')", 0.0005, "0.001;"}, - {"(F5.3,';')", 0.00049999, "0.000;"}, + {"(F5.3,';')", + 0.0005000000000000000104083408558608425664715468883514404296875, + "0.001;"}, + {"(F5.3,';')", + 0.000499999999999999901988123607310399165726266801357269287109375, + "0.000;"}, {"(F5.3,';')", 0.000099999, "0.000;"}, {"(F5.3,';')", -99.999, "*****;"}, {"(F5.3,';')", -9.9999, "*****;"}, @@ -719,17 +723,30 @@ TEST(IOApiTests, FormatDoubleValues) { {"(F5.3,';')", -0.099999, "-.100;"}, {"(F5.3,';')", -0.0099999, "-.010;"}, {"(F5.3,';')", -0.00099999, "-.001;"}, - {"(F5.3,';')", -0.0005, "-.001;"}, - {"(F5.3,';')", -0.00049999, "-.000;"}, + {"(F5.3,';')", + -0.0005000000000000000104083408558608425664715468883514404296875, + "-.001;"}, + {"(F5.3,';')", + -0.000499999999999999901988123607310399165726266801357269287109375, + "-.000;"}, {"(F5.3,';')", -0.000099999, "-.000;"}, {"(F0.1,';')", 0.0, ".0;"}, + {"(F5.0,';')", -0.5000000000000001, " -1.;"}, + {"(F5.0,';')", -0.5, " -0.;"}, + {"(F5.0,';')", -0.49999999999999994, " -0.;"}, + {"(F5.0,';')", 0.49999999999999994, " 0.;"}, + {"(F5.0,';')", 0.5, " 0.;"}, + {"(F5.0,';')", 0.5000000000000001, " 1.;"}, }; for (auto const &[format, value, expect] : individualTestCases) { std::string got; + char hex[17]; + std::snprintf(hex, sizeof hex, "%016llx", + *reinterpret_cast(&value)); ASSERT_TRUE(CompareFormatReal(format, value, expect, got)) - << "Failed to format " << format << ", expected '" << expect - << "', got '" << got << "'"; + << "Failed to format " << value << " 0x" << hex << " with format " + << format << ", expected '" << expect << "', got '" << got << "'"; } // Problematic EN formatting edge cases with rounding diff --git a/libc/src/__support/FPUtil/FPBits.h b/libc/src/__support/FPUtil/FPBits.h index ca98aa71262491031e6b077972904239d2f41035..bd075fe3d72872bade2f4acbb7afda9cc45b1c1d 100644 --- a/libc/src/__support/FPUtil/FPBits.h +++ b/libc/src/__support/FPUtil/FPBits.h @@ -59,13 +59,13 @@ template struct FPBits { return bits & FloatProp::MANTISSA_MASK; } - LIBC_INLINE constexpr void set_unbiased_exponent(UIntType expVal) { + LIBC_INLINE constexpr void set_biased_exponent(UIntType expVal) { expVal = (expVal << (FloatProp::MANTISSA_WIDTH)) & FloatProp::EXPONENT_MASK; bits &= ~(FloatProp::EXPONENT_MASK); bits |= expVal; } - LIBC_INLINE constexpr uint16_t get_unbiased_exponent() const { + LIBC_INLINE constexpr uint16_t get_biased_exponent() const { return uint16_t((bits & FloatProp::EXPONENT_MASK) >> (FloatProp::MANTISSA_WIDTH)); } @@ -73,7 +73,7 @@ template struct FPBits { // The function return mantissa with the implicit bit set iff the current // value is a valid normal number. LIBC_INLINE constexpr UIntType get_explicit_mantissa() { - return ((get_unbiased_exponent() > 0 && !is_inf_or_nan()) + return ((get_biased_exponent() > 0 && !is_inf_or_nan()) ? (FloatProp::MANTISSA_MASK + 1) : 0) | (FloatProp::MANTISSA_MASK & bits); @@ -126,7 +126,7 @@ template struct FPBits { LIBC_INLINE constexpr UIntType uintval() const { return bits; } LIBC_INLINE constexpr int get_exponent() const { - return int(get_unbiased_exponent()) - EXPONENT_BIAS; + return int(get_biased_exponent()) - EXPONENT_BIAS; } // If the number is subnormal, the exponent is treated as if it were the @@ -136,13 +136,13 @@ template struct FPBits { // will give a slightly incorrect result. Additionally, zero has an exponent // of zero, and that should actually be treated as zero. LIBC_INLINE constexpr int get_explicit_exponent() const { - const int unbiased_exp = int(get_unbiased_exponent()); + const int biased_exp = int(get_biased_exponent()); if (is_zero()) { return 0; - } else if (unbiased_exp == 0) { + } else if (biased_exp == 0) { return 1 - EXPONENT_BIAS; } else { - return unbiased_exp - EXPONENT_BIAS; + return biased_exp - EXPONENT_BIAS; } } @@ -228,7 +228,7 @@ template struct FPBits { if (LIBC_LIKELY(ep >= 0)) { // Implicit number bit will be removed by mask result.set_mantissa(number); - result.set_unbiased_exponent(ep + 1); + result.set_biased_exponent(ep + 1); } else { result.set_mantissa(number >> -ep); } @@ -236,10 +236,10 @@ template struct FPBits { } LIBC_INLINE static constexpr FPBits - create_value(bool sign, UIntType unbiased_exp, UIntType mantissa) { + create_value(bool sign, UIntType biased_exp, UIntType mantissa) { FPBits result; result.set_sign(sign); - result.set_unbiased_exponent(unbiased_exp); + result.set_biased_exponent(biased_exp); result.set_mantissa(mantissa); return result; } diff --git a/libc/src/__support/FPUtil/FloatProperties.h b/libc/src/__support/FPUtil/FloatProperties.h index 59e261e1047f1ada1a601b0149bd5e407f5f593f..ef6a3cd403db61b03b74f0e39cf46c175f8f6c52 100644 --- a/libc/src/__support/FPUtil/FloatProperties.h +++ b/libc/src/__support/FPUtil/FloatProperties.h @@ -10,7 +10,7 @@ #define LLVM_LIBC_SRC___SUPPORT_FPUTIL_FLOATPROPERTIES_H #include "src/__support/UInt128.h" -#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/attributes.h" // LIBC_INLINE, LIBC_INLINE_VAR #include "src/__support/macros/properties/float.h" // LIBC_COMPILER_HAS_FLOAT128 #include @@ -27,116 +27,177 @@ enum class FPType { X86_Binary80, }; -template struct FPProperties {}; -template <> struct FPProperties { - typedef uint32_t BitsType; +// For now 'FPEncoding', 'FPBaseProperties' and 'FPCommonProperties' are +// implementation details. +namespace internal { - static constexpr uint32_t BIT_WIDTH = sizeof(BitsType) * 8; - - static constexpr uint32_t MANTISSA_WIDTH = 23; - // The mantissa precision includes the implicit bit. - static constexpr uint32_t MANTISSA_PRECISION = MANTISSA_WIDTH + 1; - static constexpr uint32_t EXPONENT_WIDTH = 8; - static constexpr BitsType MANTISSA_MASK = (BitsType(1) << MANTISSA_WIDTH) - 1; - static constexpr BitsType SIGN_MASK = BitsType(1) - << (EXPONENT_WIDTH + MANTISSA_WIDTH); - static constexpr BitsType EXPONENT_MASK = ~(SIGN_MASK | MANTISSA_MASK); - static constexpr uint32_t EXPONENT_BIAS = 127; - - static constexpr BitsType EXP_MANT_MASK = MANTISSA_MASK + EXPONENT_MASK; - static_assert(EXP_MANT_MASK == ~SIGN_MASK, - "Exponent and mantissa masks are not as expected."); - - // If a number x is a NAN, then it is a quiet NAN if: - // QuietNaNMask & bits(x) != 0 - // Else, it is a signalling NAN. - static constexpr BitsType QUIET_NAN_MASK = 0x00400000U; +// The type of encoding for supported floating point types. +enum class FPEncoding { + IEEE754, + X86_ExtendedPrecision, }; -template <> struct FPProperties { - typedef uint64_t BitsType; - - static constexpr uint32_t BIT_WIDTH = sizeof(BitsType) * 8; - - static constexpr uint32_t MANTISSA_WIDTH = 52; - static constexpr uint32_t MANTISSA_PRECISION = MANTISSA_WIDTH + 1; - static constexpr uint32_t EXPONENT_WIDTH = 11; - static constexpr BitsType MANTISSA_MASK = (BitsType(1) << MANTISSA_WIDTH) - 1; - static constexpr BitsType SIGN_MASK = BitsType(1) - << (EXPONENT_WIDTH + MANTISSA_WIDTH); - static constexpr BitsType EXPONENT_MASK = ~(SIGN_MASK | MANTISSA_MASK); - static constexpr uint32_t EXPONENT_BIAS = 1023; +template struct FPBaseProperties {}; - static constexpr BitsType EXP_MANT_MASK = MANTISSA_MASK + EXPONENT_MASK; - static_assert(EXP_MANT_MASK == ~SIGN_MASK, - "Exponent and mantissa masks are not as expected."); +template <> struct FPBaseProperties { + using UIntType = uint16_t; + LIBC_INLINE_VAR static constexpr int TOTAL_BITS = 16; + LIBC_INLINE_VAR static constexpr int SIG_BITS = 10; + LIBC_INLINE_VAR static constexpr int EXP_BITS = 5; + LIBC_INLINE_VAR static constexpr auto ENCODING = FPEncoding::IEEE754; +}; - // If a number x is a NAN, then it is a quiet NAN if: - // QuietNaNMask & bits(x) != 0 - // Else, it is a signalling NAN. - static constexpr BitsType QUIET_NAN_MASK = 0x0008000000000000ULL; +template <> struct FPBaseProperties { + using UIntType = uint32_t; + LIBC_INLINE_VAR static constexpr int TOTAL_BITS = 32; + LIBC_INLINE_VAR static constexpr int SIG_BITS = 23; + LIBC_INLINE_VAR static constexpr int EXP_BITS = 8; + LIBC_INLINE_VAR static constexpr auto ENCODING = FPEncoding::IEEE754; }; -// Properties for numbers represented in 80 bits long double on non-Windows x86 -// platforms. -template <> struct FPProperties { - typedef UInt128 BitsType; +template <> struct FPBaseProperties { + using UIntType = uint64_t; + LIBC_INLINE_VAR static constexpr int TOTAL_BITS = 64; + LIBC_INLINE_VAR static constexpr int SIG_BITS = 52; + LIBC_INLINE_VAR static constexpr int EXP_BITS = 11; + LIBC_INLINE_VAR static constexpr auto ENCODING = FPEncoding::IEEE754; +}; - static constexpr uint32_t BIT_WIDTH = (sizeof(BitsType) * 8) - 48; - static constexpr BitsType FULL_WIDTH_MASK = ((BitsType(1) << BIT_WIDTH) - 1); +template <> struct FPBaseProperties { + using UIntType = UInt128; + LIBC_INLINE_VAR static constexpr int TOTAL_BITS = 128; + LIBC_INLINE_VAR static constexpr int SIG_BITS = 112; + LIBC_INLINE_VAR static constexpr int EXP_BITS = 15; + LIBC_INLINE_VAR static constexpr auto ENCODING = FPEncoding::IEEE754; +}; - static constexpr uint32_t MANTISSA_WIDTH = 63; - static constexpr uint32_t MANTISSA_PRECISION = MANTISSA_WIDTH + 1; - static constexpr uint32_t EXPONENT_WIDTH = 15; - static constexpr BitsType MANTISSA_MASK = (BitsType(1) << MANTISSA_WIDTH) - 1; +template <> struct FPBaseProperties { + using UIntType = UInt128; + LIBC_INLINE_VAR static constexpr int TOTAL_BITS = 80; + LIBC_INLINE_VAR static constexpr int SIG_BITS = 64; + LIBC_INLINE_VAR static constexpr int EXP_BITS = 15; + LIBC_INLINE_VAR static constexpr auto ENCODING = + FPEncoding::X86_ExtendedPrecision; +}; - // The x86 80 bit float represents the leading digit of the mantissa - // explicitly. This is the mask for that bit. - static constexpr BitsType EXPLICIT_BIT_MASK = (BitsType(1) << MANTISSA_WIDTH); +// TODO: Move this utility elsewhere. +template static constexpr T mask_trailing_ones() { + static_assert(cpp::is_unsigned_v); + constexpr unsigned t_bits = CHAR_BIT * sizeof(T); + static_assert(count <= t_bits && "Invalid bit index"); + // It's important not to initialize T with -1, since T may be BigInt which + // will take -1 as a uint64_t and only initialize the low 64 bits. + return count == 0 ? 0 : ((~T(0)) >> (t_bits - count)); +} - static constexpr BitsType SIGN_MASK = - BitsType(1) << (EXPONENT_WIDTH + MANTISSA_WIDTH + 1); - static constexpr BitsType EXPONENT_MASK = - ((BitsType(1) << EXPONENT_WIDTH) - 1) << (MANTISSA_WIDTH + 1); - static constexpr uint32_t EXPONENT_BIAS = 16383; +// Derives more properties from 'FPBaseProperties' above. +// This class serves as a halfway point between 'FPBaseProperties' and +// 'FPProperties' below. +template +struct FPCommonProperties : private FPBaseProperties { +private: + using UP = FPBaseProperties; + using UP::EXP_BITS; + using UP::SIG_BITS; + using UP::TOTAL_BITS; + using UIntType = typename UP::UIntType; + + LIBC_INLINE_VAR static constexpr int STORAGE_BITS = + sizeof(UIntType) * CHAR_BIT; + static_assert(STORAGE_BITS >= TOTAL_BITS); + + // The number of bits to represent sign. + // For documentation purpose, always 1. + LIBC_INLINE_VAR static constexpr int SIGN_BITS = 1; + static_assert(SIGN_BITS + EXP_BITS + SIG_BITS == TOTAL_BITS); + + // The exponent bias. Always positive. + LIBC_INLINE_VAR static constexpr int32_t EXP_BIAS = + (1U << (EXP_BITS - 1U)) - 1U; + static_assert(EXP_BIAS > 0); + + // Shifts + LIBC_INLINE_VAR static constexpr int SIG_MASK_SHIFT = 0; + LIBC_INLINE_VAR static constexpr int EXP_MASK_SHIFT = SIG_BITS; + LIBC_INLINE_VAR static constexpr int SIGN_MASK_SHIFT = SIG_BITS + EXP_BITS; + + // Masks + LIBC_INLINE_VAR static constexpr UIntType SIG_MASK = + mask_trailing_ones() << SIG_MASK_SHIFT; + LIBC_INLINE_VAR static constexpr UIntType EXP_MASK = + mask_trailing_ones() << EXP_MASK_SHIFT; + // Trailing underscore on SIGN_MASK_ is temporary - it will be removed + // once we can replace the public part below with the private one. + LIBC_INLINE_VAR static constexpr UIntType SIGN_MASK_ = + mask_trailing_ones() << SIGN_MASK_SHIFT; + LIBC_INLINE_VAR static constexpr UIntType FP_MASK = + mask_trailing_ones(); + static_assert((SIG_MASK & EXP_MASK & SIGN_MASK_) == 0, "masks disjoint"); + static_assert((SIG_MASK | EXP_MASK | SIGN_MASK_) == FP_MASK, "masks cover"); + + LIBC_INLINE static constexpr UIntType bit_at(int position) { + return UIntType(1) << position; + } - static constexpr BitsType EXP_MANT_MASK = - MANTISSA_MASK | EXPLICIT_BIT_MASK | EXPONENT_MASK; - static_assert(EXP_MANT_MASK == (~SIGN_MASK & FULL_WIDTH_MASK), - "Exponent and mantissa masks are not as expected."); + LIBC_INLINE_VAR static constexpr UIntType QNAN_MASK = + UP::ENCODING == FPEncoding::X86_ExtendedPrecision + ? bit_at(SIG_BITS - 1) | bit_at(SIG_BITS - 2) // 0b1100... + : bit_at(SIG_BITS - 1); // 0b1000... + + LIBC_INLINE_VAR static constexpr UIntType SNAN_MASK = + UP::ENCODING == FPEncoding::X86_ExtendedPrecision + ? bit_at(SIG_BITS - 1) | bit_at(SIG_BITS - 3) // 0b1010... + : bit_at(SIG_BITS - 2); // 0b0100... + + // The number of bits after the decimal dot when the number if in normal form. + LIBC_INLINE_VAR static constexpr int FRACTION_BITS = + UP::ENCODING == FPEncoding::X86_ExtendedPrecision ? SIG_BITS - 1 + : SIG_BITS; + +public: + // Public facing API to keep the change local to this file. + using BitsType = UIntType; + + LIBC_INLINE_VAR static constexpr uint32_t BIT_WIDTH = TOTAL_BITS; + LIBC_INLINE_VAR static constexpr uint32_t MANTISSA_WIDTH = FRACTION_BITS; + LIBC_INLINE_VAR static constexpr uint32_t MANTISSA_PRECISION = + MANTISSA_WIDTH + 1; + LIBC_INLINE_VAR static constexpr BitsType MANTISSA_MASK = + mask_trailing_ones(); + LIBC_INLINE_VAR static constexpr uint32_t EXPONENT_WIDTH = EXP_BITS; + LIBC_INLINE_VAR static constexpr uint32_t EXPONENT_BIAS = + static_cast(EXP_BIAS); + LIBC_INLINE_VAR static constexpr BitsType SIGN_MASK = SIGN_MASK_; + LIBC_INLINE_VAR static constexpr BitsType EXPONENT_MASK = EXP_MASK; + LIBC_INLINE_VAR static constexpr BitsType EXP_MANT_MASK = EXP_MASK | SIG_MASK; // If a number x is a NAN, then it is a quiet NAN if: // QuietNaNMask & bits(x) != 0 // Else, it is a signalling NAN. - static constexpr BitsType QUIET_NAN_MASK = BitsType(1) - << (MANTISSA_WIDTH - 1); + static constexpr BitsType QUIET_NAN_MASK = QNAN_MASK; }; -// Properties for numbers represented in 128 bits long double on non x86 -// platform. -template <> struct FPProperties { - typedef UInt128 BitsType; +} // namespace internal - static constexpr uint32_t BIT_WIDTH = sizeof(BitsType) << 3; +template +struct FPProperties : public internal::FPCommonProperties {}; - static constexpr uint32_t MANTISSA_WIDTH = 112; - static constexpr uint32_t MANTISSA_PRECISION = MANTISSA_WIDTH + 1; - static constexpr uint32_t EXPONENT_WIDTH = 15; - static constexpr BitsType MANTISSA_MASK = (BitsType(1) << MANTISSA_WIDTH) - 1; - static constexpr BitsType SIGN_MASK = BitsType(1) - << (EXPONENT_WIDTH + MANTISSA_WIDTH); - static constexpr BitsType EXPONENT_MASK = ~(SIGN_MASK | MANTISSA_MASK); - static constexpr uint32_t EXPONENT_BIAS = 16383; +// ---------------- +// Work In Progress +// ---------------- +// The 'FPProperties' template specializations below are being slowly replaced +// with properties from 'FPCommonProperties' above. Once specializations are +// empty, 'FPProperties' declaration can be fully replace with +// 'FPCommonProperties' implementation. - static constexpr BitsType EXP_MANT_MASK = MANTISSA_MASK | EXPONENT_MASK; - static_assert(EXP_MANT_MASK == ~SIGN_MASK, - "Exponent and mantissa masks are not as expected."); - - // If a number x is a NAN, then it is a quiet NAN if: - // QuietNaNMask & bits(x) != 0 - // Else, it is a signalling NAN. - static constexpr BitsType QUIET_NAN_MASK = BitsType(1) - << (MANTISSA_WIDTH - 1); +// Properties for numbers represented in 80 bits long double on non-Windows x86 +// platforms. +template <> +struct FPProperties + : public internal::FPCommonProperties { + // The x86 80 bit float represents the leading digit of the mantissa + // explicitly. This is the mask for that bit. + static constexpr BitsType EXPLICIT_BIT_MASK = (BitsType(1) << MANTISSA_WIDTH); }; //----------------------------------------------------------------------------- diff --git a/libc/src/__support/FPUtil/Hypot.h b/libc/src/__support/FPUtil/Hypot.h index 8a6eb4b920acd60316b009fa5abe85b78a0cb946..42d9e1b3f8cec5c96458ec1822da1fe3a830ae71 100644 --- a/libc/src/__support/FPUtil/Hypot.h +++ b/libc/src/__support/FPUtil/Hypot.h @@ -120,8 +120,8 @@ LIBC_INLINE T hypot(T x, T y) { return y; } - uint16_t x_exp = x_bits.get_unbiased_exponent(); - uint16_t y_exp = y_bits.get_unbiased_exponent(); + uint16_t x_exp = x_bits.get_biased_exponent(); + uint16_t y_exp = y_bits.get_biased_exponent(); uint16_t exp_diff = (x_exp > y_exp) ? (x_exp - y_exp) : (y_exp - x_exp); if ((exp_diff >= MantissaWidth::VALUE + 2) || (x == 0) || (y == 0)) { @@ -178,7 +178,7 @@ LIBC_INLINE T hypot(T x, T y) { // But before that, remember to store the losing bits to sticky. // The shift length is for a^2 and b^2, so it's double of the exponent // difference between a and b. - uint16_t shift_length = 2 * (a_exp - b_exp); + uint16_t shift_length = static_cast(2 * (a_exp - b_exp)); sticky_bits = ((b_mant_sq & ((DUIntType(1) << shift_length) - DUIntType(1))) != DUIntType(0)); diff --git a/libc/src/__support/FPUtil/ManipulationFunctions.h b/libc/src/__support/FPUtil/ManipulationFunctions.h index 9d3fd075be47111a315827355fa72bdf38bf08ac..08adb074b121fa1564abb04a0f99b54cc137b5f5 100644 --- a/libc/src/__support/FPUtil/ManipulationFunctions.h +++ b/libc/src/__support/FPUtil/ManipulationFunctions.h @@ -144,10 +144,11 @@ LIBC_INLINE T ldexp(T x, int exp) { return normal; } -template < - typename T, typename U, - cpp::enable_if_t && cpp::is_floating_point_v, - int> = 0> +template && + cpp::is_floating_point_v && + (sizeof(T) <= sizeof(U)), + int> = 0> LIBC_INLINE T nextafter(T from, U to) { FPBits from_bits(from); if (from_bits.is_nan()) @@ -157,6 +158,9 @@ LIBC_INLINE T nextafter(T from, U to) { if (to_bits.is_nan()) return static_cast(to); + // NOTE: This would work only if `U` has a greater or equal precision than + // `T`. Otherwise `from` could loose its precision and the following statement + // could incorrectly evaluate to `true`. if (static_cast(from) == to) return static_cast(to); diff --git a/libc/src/__support/FPUtil/NormalFloat.h b/libc/src/__support/FPUtil/NormalFloat.h index d59de14fb695e86e2bb2f0bfdb5fdb1b2c593afa..d3236316a87995165f6aa2ad1e03e561b5c936bf 100644 --- a/libc/src/__support/FPUtil/NormalFloat.h +++ b/libc/src/__support/FPUtil/NormalFloat.h @@ -111,7 +111,7 @@ template struct NormalFloat { const UIntType shift_out_mask = (UIntType(1) << shift) - 1; const UIntType shift_out_value = mantissa & shift_out_mask; const UIntType halfway_value = UIntType(1) << (shift - 1); - result.set_unbiased_exponent(0); + result.set_biased_exponent(0); result.set_mantissa(mantissa >> shift); UIntType new_mantissa = result.get_mantissa(); if (shift_out_value > halfway_value) { @@ -126,14 +126,14 @@ template struct NormalFloat { // mantissa was all ones (0b111..11). For such a case, we will carry // the overflow into the exponent. if (new_mantissa == ONE) - result.set_unbiased_exponent(1); + result.set_biased_exponent(1); return T(result); } else { return T(result); } } - result.set_unbiased_exponent(exponent + FPBits::EXPONENT_BIAS); + result.set_biased_exponent(exponent + FPBits::EXPONENT_BIAS); result.set_mantissa(mantissa); return T(result); } @@ -151,12 +151,12 @@ private: } // Normalize subnormal numbers. - if (bits.get_unbiased_exponent() == 0) { + if (bits.get_biased_exponent() == 0) { unsigned shift = evaluate_normalization_shift(bits.get_mantissa()); mantissa = UIntType(bits.get_mantissa()) << shift; exponent = 1 - FPBits::EXPONENT_BIAS - shift; } else { - exponent = bits.get_unbiased_exponent() - FPBits::EXPONENT_BIAS; + exponent = bits.get_biased_exponent() - FPBits::EXPONENT_BIAS; mantissa = ONE | bits.get_mantissa(); } } @@ -184,7 +184,7 @@ NormalFloat::init_from_bits(FPBits bits) { return; } - if (bits.get_unbiased_exponent() == 0) { + if (bits.get_biased_exponent() == 0) { if (bits.get_implicit_bit() == 0) { // Since we ignore zero value, the mantissa in this case is non-zero. int normalization_shift = @@ -201,7 +201,7 @@ NormalFloat::init_from_bits(FPBits bits) { exponent = 0; mantissa = 0; } else { - exponent = bits.get_unbiased_exponent() - 16383; + exponent = bits.get_biased_exponent() - 16383; mantissa = ONE | bits.get_mantissa(); } } @@ -228,7 +228,7 @@ template <> LIBC_INLINE NormalFloat::operator long double() const { const UIntType shift_out_mask = (UIntType(1) << shift) - 1; const UIntType shift_out_value = mantissa & shift_out_mask; const UIntType halfway_value = UIntType(1) << (shift - 1); - result.set_unbiased_exponent(0); + result.set_biased_exponent(0); result.set_mantissa(mantissa >> shift); UIntType new_mantissa = result.get_mantissa(); if (shift_out_value > halfway_value) { @@ -243,7 +243,7 @@ template <> LIBC_INLINE NormalFloat::operator long double() const { // mantissa was all ones (0b111..11). For such a case, we will carry // the overflow into the exponent and set the implicit bit to 1. if (new_mantissa == ONE) { - result.set_unbiased_exponent(1); + result.set_biased_exponent(1); result.set_implicit_bit(1); } else { result.set_implicit_bit(0); @@ -254,7 +254,7 @@ template <> LIBC_INLINE NormalFloat::operator long double() const { } } - result.set_unbiased_exponent(biased_exponent); + result.set_biased_exponent(biased_exponent); result.set_mantissa(mantissa); result.set_implicit_bit(1); return static_cast(result); diff --git a/libc/src/__support/FPUtil/dyadic_float.h b/libc/src/__support/FPUtil/dyadic_float.h index b7920943804e6106da8e986b741203247e76081d..5f0d8f49ccf64dc2911ec75f5b1962993170ea2e 100644 --- a/libc/src/__support/FPUtil/dyadic_float.h +++ b/libc/src/__support/FPUtil/dyadic_float.h @@ -94,7 +94,7 @@ template struct DyadicFloat { return 0.0; // Assume that it is normalized, and output is also normal. - constexpr size_t PRECISION = FloatProperties::MANTISSA_WIDTH + 1; + constexpr uint32_t PRECISION = FloatProperties::MANTISSA_PRECISION; using output_bits_t = typename FPBits::UIntType; int exp_hi = exponent + static_cast((Bits - 1) + @@ -110,12 +110,12 @@ template struct DyadicFloat { exp_hi = FloatProperties::EXPONENT_BIAS; } - int exp_lo = exp_hi - PRECISION - 1; + int exp_lo = exp_hi - static_cast(PRECISION) - 1; MantissaType m_hi(mantissa >> shift); T d_hi = FPBits::create_value(sign, exp_hi, - output_bits_t(m_hi) & + static_cast(m_hi) & FloatProperties::MANTISSA_MASK) .get_val(); diff --git a/libc/src/__support/FPUtil/fpbits_str.h b/libc/src/__support/FPUtil/fpbits_str.h index 4dec85ac2cc8624784de2c74661762992df29002..5d0bb6cf1ac4d96ce4202317e3b95dcbc1d0d3fd 100644 --- a/libc/src/__support/FPUtil/fpbits_str.h +++ b/libc/src/__support/FPUtil/fpbits_str.h @@ -53,7 +53,7 @@ template LIBC_INLINE cpp::string str(fputil::FPBits x) { s += sign_char(x.get_sign()); s += ", E: "; - const details::ZeroPaddedHexFmt exponent(x.get_unbiased_exponent()); + const details::ZeroPaddedHexFmt exponent(x.get_biased_exponent()); s += exponent.view(); if constexpr (cpp::is_same_v && diff --git a/libc/src/__support/FPUtil/generic/FMA.h b/libc/src/__support/FPUtil/generic/FMA.h index 4a825f2f4942cb05c29c0d992a6251e95651e94c..61a1401c30e827c63882c7f3fdf0ceec2a593124 100644 --- a/libc/src/__support/FPUtil/generic/FMA.h +++ b/libc/src/__support/FPUtil/generic/FMA.h @@ -58,7 +58,7 @@ template <> LIBC_INLINE float fma(float x, float y, float z) { // bit of sum, so that the sticky bits used when rounding sum to float are // correct (when it matters). fputil::FPBits t( - (bit_prod.get_unbiased_exponent() >= bitz.get_unbiased_exponent()) + (bit_prod.get_biased_exponent() >= bitz.get_biased_exponent()) ? ((double(bit_sum) - double(bit_prod)) - double(bitz)) : ((double(bit_sum) - double(bitz)) - double(bit_prod))); @@ -106,15 +106,15 @@ template <> LIBC_INLINE double fma(double x, double y, double z) { int z_exp = 0; // Normalize denormal inputs. - if (LIBC_UNLIKELY(FPBits(x).get_unbiased_exponent() == 0)) { + if (LIBC_UNLIKELY(FPBits(x).get_biased_exponent() == 0)) { x_exp -= 52; x *= 0x1.0p+52; } - if (LIBC_UNLIKELY(FPBits(y).get_unbiased_exponent() == 0)) { + if (LIBC_UNLIKELY(FPBits(y).get_biased_exponent() == 0)) { y_exp -= 52; y *= 0x1.0p+52; } - if (LIBC_UNLIKELY(FPBits(z).get_unbiased_exponent() == 0)) { + if (LIBC_UNLIKELY(FPBits(z).get_biased_exponent() == 0)) { z_exp -= 52; z *= 0x1.0p+52; } @@ -124,9 +124,9 @@ template <> LIBC_INLINE double fma(double x, double y, double z) { bool y_sign = y_bits.get_sign(); bool z_sign = z_bits.get_sign(); bool prod_sign = x_sign != y_sign; - x_exp += x_bits.get_unbiased_exponent(); - y_exp += y_bits.get_unbiased_exponent(); - z_exp += z_bits.get_unbiased_exponent(); + x_exp += x_bits.get_biased_exponent(); + y_exp += y_bits.get_biased_exponent(); + z_exp += z_bits.get_biased_exponent(); if (LIBC_UNLIKELY(x_exp == FPBits::MAX_EXPONENT || y_exp == FPBits::MAX_EXPONENT || diff --git a/libc/src/__support/FPUtil/generic/FMod.h b/libc/src/__support/FPUtil/generic/FMod.h index 0e71b039d5c061fcade464b1c0de7233ae8c7853..7502660c88a133da54b71fc870c62318c6f36374 100644 --- a/libc/src/__support/FPUtil/generic/FMod.h +++ b/libc/src/__support/FPUtil/generic/FMod.h @@ -233,8 +233,8 @@ private: return FPB(FPB::zero()); // |x|=|y| return 0.0 } - int e_x = sx.get_unbiased_exponent(); - int e_y = sy.get_unbiased_exponent(); + int e_x = sx.get_biased_exponent(); + int e_y = sy.get_biased_exponent(); // Most common case where |y| is "very normal" and |x/y| < 2^EXPONENT_WIDTH if (LIBC_LIKELY(e_y > int(FPB::FloatProp::MANTISSA_WIDTH) && diff --git a/libc/src/__support/FPUtil/generic/sqrt.h b/libc/src/__support/FPUtil/generic/sqrt.h index 6ae2171bacf7b84a00063e0d43e72f52f4844e32..5bde9589fdc012c5686656f21d3d7c515f2b5a5a 100644 --- a/libc/src/__support/FPUtil/generic/sqrt.h +++ b/libc/src/__support/FPUtil/generic/sqrt.h @@ -97,7 +97,7 @@ LIBC_INLINE cpp::enable_if_t, T> sqrt(T x) { UIntType x_mant = bits.get_mantissa(); // Step 1a: Normalize denormal input and append hidden bit to the mantissa - if (bits.get_unbiased_exponent() == 0) { + if (bits.get_biased_exponent() == 0) { ++x_exp; // let x_exp be the correct exponent of ONE bit. internal::normalize(x_exp, x_mant); } else { diff --git a/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h b/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h index 0e7907e82943a5aa025a537973ecde34e4bacca2..2f25be54e0bc3626085687fac6b3ae50e1942eb5 100644 --- a/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h +++ b/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h @@ -65,7 +65,7 @@ LIBC_INLINE long double sqrt(long double x) { // Step 1a: Normalize denormal input if (bits.get_implicit_bit()) { x_mant |= ONE; - } else if (bits.get_unbiased_exponent() == 0) { + } else if (bits.get_biased_exponent() == 0) { normalize(x_exp, x_mant); } @@ -128,7 +128,7 @@ LIBC_INLINE long double sqrt(long double x) { // Extract output FPBits out(0.0L); - out.set_unbiased_exponent(x_exp); + out.set_biased_exponent(x_exp); out.set_implicit_bit(1); out.set_mantissa((y & (ONE - 1))); diff --git a/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h b/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h index 2f07ff4a2c3e500f198483bc7a770c33812e5c01..bbc30ff7a376dfc3d096daca3704259466f1f495 100644 --- a/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h +++ b/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h @@ -71,7 +71,7 @@ template <> struct FPBits { return bits & (FloatProp::MANTISSA_MASK | FloatProp::EXPLICIT_BIT_MASK); } - LIBC_INLINE constexpr void set_unbiased_exponent(UIntType expVal) { + LIBC_INLINE constexpr void set_biased_exponent(UIntType expVal) { expVal = (expVal << (FloatProp::BIT_WIDTH - 1 - FloatProp::EXPONENT_WIDTH)) & FloatProp::EXPONENT_MASK; @@ -79,7 +79,7 @@ template <> struct FPBits { bits |= expVal; } - LIBC_INLINE constexpr uint16_t get_unbiased_exponent() const { + LIBC_INLINE constexpr uint16_t get_biased_exponent() const { return uint16_t((bits & FloatProp::EXPONENT_MASK) >> (FloatProp::BIT_WIDTH - 1 - FloatProp::EXPONENT_WIDTH)); } @@ -137,7 +137,7 @@ template <> struct FPBits { } LIBC_INLINE constexpr int get_exponent() const { - return int(get_unbiased_exponent()) - EXPONENT_BIAS; + return int(get_biased_exponent()) - EXPONENT_BIAS; } // If the number is subnormal, the exponent is treated as if it were the @@ -147,38 +147,38 @@ template <> struct FPBits { // will give a slightly incorrect result. Additionally, zero has an exponent // of zero, and that should actually be treated as zero. LIBC_INLINE constexpr int get_explicit_exponent() const { - const int unbiased_exp = int(get_unbiased_exponent()); + const int biased_exp = int(get_biased_exponent()); if (is_zero()) { return 0; - } else if (unbiased_exp == 0) { + } else if (biased_exp == 0) { return 1 - EXPONENT_BIAS; } else { - return unbiased_exp - EXPONENT_BIAS; + return biased_exp - EXPONENT_BIAS; } } LIBC_INLINE constexpr bool is_zero() const { - return get_unbiased_exponent() == 0 && get_mantissa() == 0 && + return get_biased_exponent() == 0 && get_mantissa() == 0 && get_implicit_bit() == 0; } LIBC_INLINE constexpr bool is_inf() const { - return get_unbiased_exponent() == MAX_EXPONENT && get_mantissa() == 0 && + return get_biased_exponent() == MAX_EXPONENT && get_mantissa() == 0 && get_implicit_bit() == 1; } LIBC_INLINE constexpr bool is_nan() const { - if (get_unbiased_exponent() == MAX_EXPONENT) { + if (get_biased_exponent() == MAX_EXPONENT) { return (get_implicit_bit() == 0) || get_mantissa() != 0; - } else if (get_unbiased_exponent() != 0) { + } else if (get_biased_exponent() != 0) { return get_implicit_bit() == 0; } return false; } LIBC_INLINE constexpr bool is_inf_or_nan() const { - return (get_unbiased_exponent() == MAX_EXPONENT) || - (get_unbiased_exponent() != 0 && get_implicit_bit() == 0); + return (get_biased_exponent() == MAX_EXPONENT) || + (get_biased_exponent() != 0 && get_implicit_bit() == 0); } // Methods below this are used by tests. @@ -189,7 +189,7 @@ template <> struct FPBits { LIBC_INLINE static constexpr long double inf(bool sign = false) { FPBits bits(0.0l); - bits.set_unbiased_exponent(MAX_EXPONENT); + bits.set_biased_exponent(MAX_EXPONENT); bits.set_implicit_bit(1); if (sign) { bits.set_sign(true); @@ -201,7 +201,7 @@ template <> struct FPBits { LIBC_INLINE static constexpr long double build_nan(UIntType v) { FPBits bits(0.0l); - bits.set_unbiased_exponent(MAX_EXPONENT); + bits.set_biased_exponent(MAX_EXPONENT); bits.set_implicit_bit(1); bits.set_mantissa(v); return bits; @@ -228,10 +228,10 @@ template <> struct FPBits { } LIBC_INLINE static constexpr FPBits - create_value(bool sign, UIntType unbiased_exp, UIntType mantissa) { + create_value(bool sign, UIntType biased_exp, UIntType mantissa) { FPBits result; result.set_sign(sign); - result.set_unbiased_exponent(unbiased_exp); + result.set_biased_exponent(biased_exp); result.set_mantissa(mantissa); return result; } diff --git a/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h b/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h index 4508671b47ee598fbf0b8ca07caf247d0ad64df7..5e32f766ad58636f3efa75119462de8380e06458 100644 --- a/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h +++ b/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h @@ -39,8 +39,8 @@ LIBC_INLINE long double nextafter(long double from, long double to) { // Convert pseudo subnormal number to normal number. if (from_bits.get_implicit_bit() == 1 && - from_bits.get_unbiased_exponent() == 0) { - from_bits.set_unbiased_exponent(1); + from_bits.get_biased_exponent() == 0) { + from_bits.set_biased_exponent(1); } using UIntType = FPBits::UIntType; @@ -59,7 +59,7 @@ LIBC_INLINE long double nextafter(long double from, long double to) { // Incrementing exponent might overflow the value to infinity, // which is what is expected. Since NaNs are handling separately, // it will never overflow "beyond" infinity. - from_bits.set_unbiased_exponent(from_bits.get_unbiased_exponent() + 1); + from_bits.set_biased_exponent(from_bits.get_biased_exponent() + 1); if (from_bits.is_inf()) raise_except_if_required(FE_OVERFLOW | FE_INEXACT); return from_bits; @@ -75,7 +75,7 @@ LIBC_INLINE long double nextafter(long double from, long double to) { from_bits.set_mantissa(MANTISSA_MASK); // from == 0 is handled separately so decrementing the exponent will not // lead to underflow. - from_bits.set_unbiased_exponent(from_bits.get_unbiased_exponent() - 1); + from_bits.set_biased_exponent(from_bits.get_biased_exponent() - 1); return from_bits; } else { --int_val; @@ -94,7 +94,7 @@ LIBC_INLINE long double nextafter(long double from, long double to) { from_bits.set_mantissa(MANTISSA_MASK); // from == 0 is handled separately so decrementing the exponent will not // lead to underflow. - from_bits.set_unbiased_exponent(from_bits.get_unbiased_exponent() - 1); + from_bits.set_biased_exponent(from_bits.get_biased_exponent() - 1); return from_bits; } else { --int_val; @@ -107,7 +107,7 @@ LIBC_INLINE long double nextafter(long double from, long double to) { // Incrementing exponent might overflow the value to infinity, // which is what is expected. Since NaNs are handling separately, // it will never overflow "beyond" infinity. - from_bits.set_unbiased_exponent(from_bits.get_unbiased_exponent() + 1); + from_bits.set_biased_exponent(from_bits.get_biased_exponent() + 1); if (from_bits.is_inf()) raise_except_if_required(FE_OVERFLOW | FE_INEXACT); return from_bits; diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h index 3bec2e3a4713024ef24ed935fcc7b9eed1e60477..f72b995f8788db5fd48887b0144aebba2b24bbb1 100644 --- a/libc/src/__support/UInt.h +++ b/libc/src/__support/UInt.h @@ -103,13 +103,20 @@ template struct BigInt { val[i] = words[i]; } - template && - sizeof(T) <= 16 && - !cpp::is_same_v>> - LIBC_INLINE constexpr explicit operator T() const { - if constexpr (sizeof(T) <= 8) - return static_cast(val[0]); + template LIBC_INLINE constexpr explicit operator T() const { + return to(); + } + template + LIBC_INLINE constexpr cpp::enable_if_t< + cpp::is_integral_v && sizeof(T) <= 8 && !cpp::is_same_v, T> + to() const { + return static_cast(val[0]); + } + template + LIBC_INLINE constexpr cpp::enable_if_t< + cpp::is_integral_v && sizeof(T) == 16, T> + to() const { // T is 128-bit. T lo = static_cast(val[0]); @@ -121,7 +128,6 @@ template struct BigInt { return lo; } } else { - // TODO: silence shift warning return static_cast((static_cast(val[1]) << 64) + lo); } } diff --git a/libc/src/__support/str_to_float.h b/libc/src/__support/str_to_float.h index ad73e93f6faa81c2adf39427245ce271421a9f5c..7ec6b9947ad2b876330ca7c004b81883b158039a 100644 --- a/libc/src/__support/str_to_float.h +++ b/libc/src/__support/str_to_float.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_STR_TO_FLOAT_H #define LLVM_LIBC_SRC___SUPPORT_STR_TO_FLOAT_H +#include "src/__support/CPP/bit.h" #include "src/__support/CPP/limits.h" #include "src/__support/CPP/optional.h" #include "src/__support/FPUtil/FEnvImpl.h" @@ -37,45 +38,6 @@ template struct FloatConvertReturn { int error = 0; }; -template LIBC_INLINE uint32_t leading_zeroes(T inputNumber) { - constexpr uint32_t BITS_IN_T = sizeof(T) * 8; - if (inputNumber == 0) { - return BITS_IN_T; - } - uint32_t cur_guess = BITS_IN_T / 2; - uint32_t range_size = BITS_IN_T / 2; - // while either shifting by curGuess does not get rid of all of the bits or - // shifting by one less also gets rid of all of the bits then we have not - // found the first bit. - while (((inputNumber >> cur_guess) > 0) || - ((inputNumber >> (cur_guess - 1)) == 0)) { - // Binary search for the first set bit - range_size /= 2; - if (range_size == 0) { - break; - } - if ((inputNumber >> cur_guess) > 0) { - cur_guess += range_size; - } else { - cur_guess -= range_size; - } - } - if (inputNumber >> cur_guess > 0) { - cur_guess++; - } - return BITS_IN_T - cur_guess; -} - -template <> -LIBC_INLINE uint32_t leading_zeroes(uint32_t inputNumber) { - return cpp::countl_zero(inputNumber); -} - -template <> -LIBC_INLINE uint32_t leading_zeroes(uint64_t inputNumber) { - return cpp::countl_zero(inputNumber); -} - LIBC_INLINE uint64_t low64(const UInt128 &num) { return static_cast(num & 0xffffffffffffffff); } @@ -92,7 +54,7 @@ template LIBC_INLINE void set_implicit_bit(fputil::FPBits &) { template <> LIBC_INLINE void set_implicit_bit(fputil::FPBits &result) { - result.set_implicit_bit(result.get_unbiased_exponent() != 0); + result.set_implicit_bit(result.get_biased_exponent() != 0); } #endif @@ -108,10 +70,11 @@ template LIBC_INLINE cpp::optional> eisel_lemire(ExpandedFloat init_num, RoundDirection round = RoundDirection::Nearest) { + using FPBits = typename fputil::FPBits; + using FloatProp = typename FPBits::FloatProp; + using UIntType = typename FPBits::UIntType; - using BitsType = typename fputil::FPBits::UIntType; - - BitsType mantissa = init_num.mantissa; + UIntType mantissa = init_num.mantissa; int32_t exp10 = init_num.exponent; constexpr uint32_t BITS_IN_MANTISSA = sizeof(mantissa) * 8; @@ -128,12 +91,11 @@ eisel_lemire(ExpandedFloat init_num, } // Normalization - uint32_t clz = leading_zeroes(mantissa); + uint32_t clz = cpp::countl_zero(mantissa); mantissa <<= clz; uint32_t exp2 = static_cast(exp10_to_exp2(exp10)) + - BITS_IN_MANTISSA + fputil::FloatProperties::EXPONENT_BIAS - - clz; + BITS_IN_MANTISSA + FloatProp::EXPONENT_BIAS - clz; // Multiplication const uint64_t *power_of_ten = @@ -150,9 +112,7 @@ eisel_lemire(ExpandedFloat init_num, // accuracy, and the most significant bit is ignored.) = 9 bits. Similarly, // it's 6 bits for floats in this case. const uint64_t halfway_constant = - (uint64_t(1) << (BITS_IN_MANTISSA - - fputil::FloatProperties::MANTISSA_WIDTH - 3)) - - 1; + (uint64_t(1) << (BITS_IN_MANTISSA - (FloatProp::MANTISSA_WIDTH + 3))) - 1; if ((high64(first_approx) & halfway_constant) == halfway_constant && low64(first_approx) + mantissa < mantissa) { UInt128 low_bits = @@ -171,12 +131,11 @@ eisel_lemire(ExpandedFloat init_num, } // Shifting to 54 bits for doubles and 25 bits for floats - BitsType msb = - static_cast(high64(final_approx) >> (BITS_IN_MANTISSA - 1)); - BitsType final_mantissa = - static_cast(high64(final_approx) >> - (msb + BITS_IN_MANTISSA - - (fputil::FloatProperties::MANTISSA_WIDTH + 3))); + UIntType msb = + static_cast(high64(final_approx) >> (BITS_IN_MANTISSA - 1)); + UIntType final_mantissa = static_cast( + high64(final_approx) >> + (msb + BITS_IN_MANTISSA - (FloatProp::MANTISSA_WIDTH + 3))); exp2 -= static_cast(1 ^ msb); // same as !msb if (round == RoundDirection::Nearest) { @@ -202,15 +161,14 @@ eisel_lemire(ExpandedFloat init_num, // From 54 to 53 bits for doubles and 25 to 24 bits for floats final_mantissa >>= 1; - if ((final_mantissa >> (fputil::FloatProperties::MANTISSA_WIDTH + 1)) > - 0) { + if ((final_mantissa >> (FloatProp::MANTISSA_WIDTH + 1)) > 0) { final_mantissa >>= 1; ++exp2; } // The if block is equivalent to (but has fewer branches than): // if exp2 <= 0 || exp2 >= 0x7FF { etc } - if (exp2 - 1 >= (1 << fputil::FloatProperties::EXPONENT_WIDTH) - 2) { + if (exp2 - 1 >= (1 << FloatProp::EXPONENT_WIDTH) - 2) { return cpp::nullopt; } @@ -225,9 +183,11 @@ template <> LIBC_INLINE cpp::optional> eisel_lemire(ExpandedFloat init_num, RoundDirection round) { - using BitsType = typename fputil::FPBits::UIntType; + using FPBits = typename fputil::FPBits; + using FloatProp = typename FPBits::FloatProp; + using UIntType = typename FPBits::UIntType; - BitsType mantissa = init_num.mantissa; + UIntType mantissa = init_num.mantissa; int32_t exp10 = init_num.exponent; constexpr uint32_t BITS_IN_MANTISSA = sizeof(mantissa) * 8; @@ -248,12 +208,11 @@ eisel_lemire(ExpandedFloat init_num, } // Normalization - uint32_t clz = leading_zeroes(mantissa); + uint32_t clz = cpp::countl_zero(mantissa); mantissa <<= clz; uint32_t exp2 = static_cast(exp10_to_exp2(exp10)) + - BITS_IN_MANTISSA + - fputil::FloatProperties::EXPONENT_BIAS - clz; + BITS_IN_MANTISSA + FloatProp::EXPONENT_BIAS - clz; // Multiplication const uint64_t *power_of_ten = @@ -290,10 +249,7 @@ eisel_lemire(ExpandedFloat init_num, // accuracy, and the most significant bit is ignored.) = 61 bits. Similarly, // it's 12 bits for 128 bit floats in this case. constexpr UInt128 HALFWAY_CONSTANT = - (UInt128(1) << (BITS_IN_MANTISSA - - fputil::FloatProperties::MANTISSA_WIDTH - - 3)) - - 1; + (UInt128(1) << (BITS_IN_MANTISSA - (FloatProp::MANTISSA_WIDTH + 3))) - 1; if ((final_approx_upper & HALFWAY_CONSTANT) == HALFWAY_CONSTANT && final_approx_lower + mantissa < mantissa) { @@ -303,10 +259,9 @@ eisel_lemire(ExpandedFloat init_num, // Shifting to 65 bits for 80 bit floats and 113 bits for 128 bit floats uint32_t msb = static_cast(final_approx_upper >> (BITS_IN_MANTISSA - 1)); - BitsType final_mantissa = + UIntType final_mantissa = final_approx_upper >> - (msb + BITS_IN_MANTISSA - - (fputil::FloatProperties::MANTISSA_WIDTH + 3)); + (msb + BITS_IN_MANTISSA - (FloatProp::MANTISSA_WIDTH + 3)); exp2 -= static_cast(1 ^ msb); // same as !msb if (round == RoundDirection::Nearest) { @@ -331,16 +286,14 @@ eisel_lemire(ExpandedFloat init_num, // From 65 to 64 bits for 80 bit floats and 113 to 112 bits for 128 bit // floats final_mantissa >>= 1; - if ((final_mantissa >> - (fputil::FloatProperties::MANTISSA_WIDTH + 1)) > 0) { + if ((final_mantissa >> (FloatProp::MANTISSA_WIDTH + 1)) > 0) { final_mantissa >>= 1; ++exp2; } // The if block is equivalent to (but has fewer branches than): // if exp2 <= 0 || exp2 >= MANTISSA_MAX { etc } - if (exp2 - 1 >= - (1 << fputil::FloatProperties::EXPONENT_WIDTH) - 2) { + if (exp2 - 1 >= (1 << FloatProp::EXPONENT_WIDTH) - 2) { return cpp::nullopt; } @@ -368,6 +321,9 @@ template LIBC_INLINE FloatConvertReturn simple_decimal_conversion(const char *__restrict numStart, RoundDirection round = RoundDirection::Nearest) { + using FPBits = typename fputil::FPBits; + using FloatProp = typename FPBits::FloatProp; + using UIntType = typename FPBits::UIntType; int32_t exp2 = 0; HighPrecisionDecimal hpd = HighPrecisionDecimal(numStart); @@ -383,16 +339,16 @@ simple_decimal_conversion(const char *__restrict numStart, // float, return inf. if (hpd.get_decimal_point() > 0 && exp10_to_exp2(hpd.get_decimal_point() - 1) > - static_cast(fputil::FloatProperties::EXPONENT_BIAS)) { - output.num = {0, fputil::FPBits::MAX_EXPONENT}; + static_cast(FloatProp::EXPONENT_BIAS)) { + output.num = {0, FPBits::MAX_EXPONENT}; output.error = ERANGE; return output; } // If the exponent is too small even for a subnormal, return 0. if (hpd.get_decimal_point() < 0 && exp10_to_exp2(-hpd.get_decimal_point()) > - static_cast(fputil::FloatProperties::EXPONENT_BIAS + - fputil::FloatProperties::MANTISSA_WIDTH)) { + static_cast(FloatProp::EXPONENT_BIAS + + FloatProp::MANTISSA_WIDTH)) { output.num = {0, 0}; output.error = ERANGE; return output; @@ -431,19 +387,18 @@ simple_decimal_conversion(const char *__restrict numStart, hpd.shift(1); // Get the biased exponent - exp2 += fputil::FloatProperties::EXPONENT_BIAS; + exp2 += FloatProp::EXPONENT_BIAS; // Handle the exponent being too large (and return inf). - if (exp2 >= fputil::FPBits::MAX_EXPONENT) { - output.num = {0, fputil::FPBits::MAX_EXPONENT}; + if (exp2 >= FPBits::MAX_EXPONENT) { + output.num = {0, FPBits::MAX_EXPONENT}; output.error = ERANGE; return output; } // Shift left to fill the mantissa - hpd.shift(fputil::FloatProperties::MANTISSA_WIDTH); - typename fputil::FPBits::UIntType final_mantissa = - hpd.round_to_integer_type::UIntType>(); + hpd.shift(FloatProp::MANTISSA_WIDTH); + UIntType final_mantissa = hpd.round_to_integer_type(); // Handle subnormals if (exp2 <= 0) { @@ -455,25 +410,23 @@ simple_decimal_conversion(const char *__restrict numStart, // Shift right one more time to compensate for the left shift to get it // between 1 and 2. hpd.shift(-1); - final_mantissa = - hpd.round_to_integer_type::UIntType>(round); + final_mantissa = hpd.round_to_integer_type(round); // Check if by shifting right we've caused this to round to a normal number. - if ((final_mantissa >> fputil::FloatProperties::MANTISSA_WIDTH) != 0) { + if ((final_mantissa >> FloatProp::MANTISSA_WIDTH) != 0) { ++exp2; } } // Check if rounding added a bit, and shift down if that's the case. - if (final_mantissa == typename fputil::FPBits::UIntType(2) - << fputil::FloatProperties::MANTISSA_WIDTH) { + if (final_mantissa == UIntType(2) << FloatProp::MANTISSA_WIDTH) { final_mantissa >>= 1; ++exp2; // Check if this rounding causes exp2 to go out of range and make the result // INF. If this is the case, then finalMantissa and exp2 are already the // correct values for an INF result. - if (exp2 >= fputil::FPBits::MAX_EXPONENT) { + if (exp2 >= FPBits::MAX_EXPONENT) { output.error = ERANGE; } } @@ -563,18 +516,20 @@ template LIBC_INLINE cpp::optional> clinger_fast_path(ExpandedFloat init_num, RoundDirection round = RoundDirection::Nearest) { + using FPBits = typename fputil::FPBits; + using FloatProp = typename FPBits::FloatProp; + using UIntType = typename FPBits::UIntType; - typename fputil::FPBits::UIntType mantissa = init_num.mantissa; + UIntType mantissa = init_num.mantissa; int32_t exp10 = init_num.exponent; - if (mantissa >> fputil::FloatProperties::MANTISSA_WIDTH > 0) { + if ((mantissa >> FloatProp::MANTISSA_WIDTH) > 0) { return cpp::nullopt; } - fputil::FPBits result; + FPBits result; T float_mantissa; - if constexpr (cpp::is_same_v::UIntType, - cpp::UInt<128>>) { + if constexpr (cpp::is_same_v>) { float_mantissa = static_cast(fputil::DyadicFloat<128>( false, 0, fputil::DyadicFloat<128>::MantissaType( @@ -584,7 +539,7 @@ clinger_fast_path(ExpandedFloat init_num, } if (exp10 == 0) { - result = fputil::FPBits(float_mantissa); + result = FPBits(float_mantissa); } if (exp10 > 0) { if (exp10 > ClingerConsts::EXACT_POWERS_OF_TEN + @@ -600,14 +555,14 @@ clinger_fast_path(ExpandedFloat init_num, if (float_mantissa > ClingerConsts::MAX_EXACT_INT) { return cpp::nullopt; } - result = fputil::FPBits(float_mantissa * - ClingerConsts::POWERS_OF_TEN_ARRAY[exp10]); + result = + FPBits(float_mantissa * ClingerConsts::POWERS_OF_TEN_ARRAY[exp10]); } else if (exp10 < 0) { if (-exp10 > ClingerConsts::EXACT_POWERS_OF_TEN) { return cpp::nullopt; } - result = fputil::FPBits(float_mantissa / - ClingerConsts::POWERS_OF_TEN_ARRAY[-exp10]); + result = + FPBits(float_mantissa / ClingerConsts::POWERS_OF_TEN_ARRAY[-exp10]); } // If the rounding mode is not nearest, then the sign of the number may affect @@ -615,15 +570,15 @@ clinger_fast_path(ExpandedFloat init_num, // calculation is redone with a negative result, and the rounding mode is used // to select the correct result. if (round != RoundDirection::Nearest) { - fputil::FPBits negative_result; + FPBits negative_result; // I'm 99% sure this will break under fast math optimizations. - negative_result = fputil::FPBits( - (-float_mantissa) * ClingerConsts::POWERS_OF_TEN_ARRAY[exp10]); + negative_result = FPBits((-float_mantissa) * + ClingerConsts::POWERS_OF_TEN_ARRAY[exp10]); // If the results are equal, then we don't need to use the rounding mode. if (T(result) != -T(negative_result)) { - fputil::FPBits lower_result; - fputil::FPBits higher_result; + FPBits lower_result; + FPBits higher_result; if (T(result) < -T(negative_result)) { lower_result = result; @@ -643,7 +598,7 @@ clinger_fast_path(ExpandedFloat init_num, ExpandedFloat output; output.mantissa = result.get_mantissa(); - output.exponent = result.get_unbiased_exponent(); + output.exponent = result.get_biased_exponent(); return output; } @@ -691,8 +646,10 @@ template LIBC_INLINE FloatConvertReturn decimal_exp_to_float(ExpandedFloat init_num, const char *__restrict numStart, bool truncated, RoundDirection round) { + using FPBits = typename fputil::FPBits; + using UIntType = typename FPBits::UIntType; - typename fputil::FPBits::UIntType mantissa = init_num.mantissa; + UIntType mantissa = init_num.mantissa; int32_t exp10 = init_num.exponent; FloatConvertReturn output; @@ -702,7 +659,7 @@ decimal_exp_to_float(ExpandedFloat init_num, const char *__restrict numStart, // float, return inf. These bounds are relatively loose, but are mostly // serving as a first pass. Some close numbers getting through is okay. if (exp10 > get_upper_bound()) { - output.num = {0, fputil::FPBits::MAX_EXPONENT}; + output.num = {0, FPBits::MAX_EXPONENT}; output.error = ERANGE; return output; } @@ -766,40 +723,39 @@ template LIBC_INLINE FloatConvertReturn binary_exp_to_float(ExpandedFloat init_num, bool truncated, RoundDirection round) { - using BitsType = typename fputil::FPBits::UIntType; + using FPBits = typename fputil::FPBits; + using FloatProp = typename FPBits::FloatProp; + using UIntType = typename FPBits::UIntType; - BitsType mantissa = init_num.mantissa; + UIntType mantissa = init_num.mantissa; int32_t exp2 = init_num.exponent; FloatConvertReturn output; // This is the number of leading zeroes a properly normalized float of type T // should have. - constexpr int32_t NUMBITS = sizeof(BitsType) * 8; - constexpr int32_t INF_EXP = - (1 << fputil::FloatProperties::EXPONENT_WIDTH) - 1; + constexpr int32_t NUMBITS = sizeof(UIntType) * 8; + constexpr int32_t INF_EXP = (1 << FloatProp::EXPONENT_WIDTH) - 1; - // Normalization step 1: Bring the leading bit to the highest bit of BitsType. - uint32_t amount_to_shift_left = leading_zeroes(mantissa); + // Normalization step 1: Bring the leading bit to the highest bit of UIntType. + uint32_t amount_to_shift_left = cpp::countl_zero(mantissa); mantissa <<= amount_to_shift_left; - // Keep exp2 representing the exponent of the lowest bit of BitsType. + // Keep exp2 representing the exponent of the lowest bit of UIntType. exp2 -= amount_to_shift_left; // biasedExponent represents the biased exponent of the most significant bit. - int32_t biased_exponent = - exp2 + NUMBITS + fputil::FPBits::EXPONENT_BIAS - 1; + int32_t biased_exponent = exp2 + NUMBITS + FPBits::EXPONENT_BIAS - 1; // Handle numbers that're too large and get squashed to inf if (biased_exponent >= INF_EXP) { // This indicates an overflow, so we make the result INF and set errno. - output.num = {0, (1 << fputil::FloatProperties::EXPONENT_WIDTH) - 1}; + output.num = {0, (1 << FloatProp::EXPONENT_WIDTH) - 1}; output.error = ERANGE; return output; } - uint32_t amount_to_shift_right = - NUMBITS - fputil::FloatProperties::MANTISSA_WIDTH - 1; + uint32_t amount_to_shift_right = NUMBITS - FloatProp::MANTISSA_WIDTH - 1; // Handle subnormals. if (biased_exponent <= 0) { @@ -814,19 +770,19 @@ LIBC_INLINE FloatConvertReturn binary_exp_to_float(ExpandedFloat init_num, } } - BitsType round_bit_mask = BitsType(1) << (amount_to_shift_right - 1); - BitsType sticky_mask = round_bit_mask - 1; + UIntType round_bit_mask = UIntType(1) << (amount_to_shift_right - 1); + UIntType sticky_mask = round_bit_mask - 1; bool round_bit = static_cast(mantissa & round_bit_mask); bool sticky_bit = static_cast(mantissa & sticky_mask) || truncated; if (amount_to_shift_right < NUMBITS) { // Shift the mantissa and clear the implicit bit. mantissa >>= amount_to_shift_right; - mantissa &= fputil::FloatProperties::MANTISSA_MASK; + mantissa &= FloatProp::MANTISSA_MASK; } else { mantissa = 0; } - bool least_significant_bit = static_cast(mantissa & BitsType(1)); + bool least_significant_bit = static_cast(mantissa & UIntType(1)); // TODO: check that this rounding behavior is correct. @@ -845,7 +801,7 @@ LIBC_INLINE FloatConvertReturn binary_exp_to_float(ExpandedFloat init_num, } } - if (mantissa > fputil::FloatProperties::MANTISSA_MASK) { + if (mantissa > FloatProp::MANTISSA_MASK) { // Rounding causes the exponent to increase. ++biased_exponent; @@ -858,8 +814,7 @@ LIBC_INLINE FloatConvertReturn binary_exp_to_float(ExpandedFloat init_num, output.error = ERANGE; } - output.num = {mantissa & fputil::FloatProperties::MANTISSA_MASK, - biased_exponent}; + output.num = {mantissa & FloatProp::MANTISSA_MASK, biased_exponent}; return output; } @@ -887,14 +842,16 @@ template LIBC_INLINE StrToNumResult> decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, RoundDirection round) { - using BitsType = typename fputil::FPBits::UIntType; + using FPBits = typename fputil::FPBits; + using UIntType = typename FPBits::UIntType; + constexpr uint32_t BASE = 10; constexpr char EXPONENT_MARKER = 'e'; bool truncated = false; bool seen_digit = false; bool after_decimal = false; - BitsType mantissa = 0; + UIntType mantissa = 0; int32_t exponent = 0; size_t index = 0; @@ -905,8 +862,8 @@ decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, // the format mantissa * (base ^ exponent) // The loop fills the mantissa with as many digits as it can hold - const BitsType bitstype_max_div_by_base = - cpp::numeric_limits::max() / BASE; + const UIntType bitstype_max_div_by_base = + cpp::numeric_limits::max() / BASE; while (true) { if (isdigit(src[index])) { uint32_t digit = src[index] - '0'; @@ -962,10 +919,10 @@ decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, // If the result is in the valid range, then we use it. The valid range is // also within the int32 range, so this prevents overflow issues. - if (temp_exponent > fputil::FPBits::MAX_EXPONENT) { - exponent = fputil::FPBits::MAX_EXPONENT; - } else if (temp_exponent < -fputil::FPBits::MAX_EXPONENT) { - exponent = -fputil::FPBits::MAX_EXPONENT; + if (temp_exponent > FPBits::MAX_EXPONENT) { + exponent = FPBits::MAX_EXPONENT; + } else if (temp_exponent < -FPBits::MAX_EXPONENT) { + exponent = -FPBits::MAX_EXPONENT; } else { exponent = static_cast(temp_exponent); } @@ -994,14 +951,16 @@ template LIBC_INLINE StrToNumResult> hexadecimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, RoundDirection round) { - using BitsType = typename fputil::FPBits::UIntType; + using FPBits = typename fputil::FPBits; + using UIntType = typename FPBits::UIntType; + constexpr uint32_t BASE = 16; constexpr char EXPONENT_MARKER = 'p'; bool truncated = false; bool seen_digit = false; bool after_decimal = false; - BitsType mantissa = 0; + UIntType mantissa = 0; int32_t exponent = 0; size_t index = 0; @@ -1012,8 +971,8 @@ hexadecimal_string_to_float(const char *__restrict src, // the format mantissa * (base ^ exponent) // The loop fills the mantissa with as many digits as it can hold - const BitsType bitstype_max_div_by_base = - cpp::numeric_limits::max() / BASE; + const UIntType bitstype_max_div_by_base = + cpp::numeric_limits::max() / BASE; while (true) { if (isalnum(src[index])) { uint32_t digit = b36_char_to_int(src[index]); @@ -1074,10 +1033,10 @@ hexadecimal_string_to_float(const char *__restrict src, // If the result is in the valid range, then we use it. The valid range is // also within the int32 range, so this prevents overflow issues. - if (temp_exponent > fputil::FPBits::MAX_EXPONENT) { - exponent = fputil::FPBits::MAX_EXPONENT; - } else if (temp_exponent < -fputil::FPBits::MAX_EXPONENT) { - exponent = -fputil::FPBits::MAX_EXPONENT; + if (temp_exponent > FPBits::MAX_EXPONENT) { + exponent = FPBits::MAX_EXPONENT; + } else if (temp_exponent < -FPBits::MAX_EXPONENT) { + exponent = -FPBits::MAX_EXPONENT; } else { exponent = static_cast(temp_exponent); } @@ -1099,8 +1058,10 @@ hexadecimal_string_to_float(const char *__restrict src, // is used as the backend for all of the string to float functions. template LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { - using BitsType = typename fputil::FPBits::UIntType; - fputil::FPBits result = fputil::FPBits(); + using FPBits = typename fputil::FPBits; + using UIntType = typename FPBits::UIntType; + + FPBits result = FPBits(); bool seen_digit = false; char sign = '+'; @@ -1164,7 +1125,7 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { } seen_digit = parse_result.parsed_len != 0; result.set_mantissa(parse_result.value.mantissa); - result.set_unbiased_exponent(parse_result.value.exponent); + result.set_biased_exponent(parse_result.value.exponent); index += parse_result.parsed_len; error = parse_result.error; } else if (tolower(src[index]) == 'n') { // NaN @@ -1172,7 +1133,7 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { tolower(src[index + 2]) == nan_string[2]) { seen_digit = true; index += 3; - BitsType nan_mantissa = 0; + UIntType nan_mantissa = 0; // this handles the case of `NaN(n-character-sequence)`, where the // n-character-sequence is made of 0 or more letters and numbers in any // order. @@ -1186,7 +1147,7 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { if (src[index] == ')') { ++index; if (isdigit(src[left_paren + 1])) { - // This is to prevent errors when BitsType is larger than 64 bits, + // This is to prevent errors when UIntType is larger than 64 bits, // since strtointeger only supports up to 64 bits. This is actually // more than is required by the specification, which says for the // input type "NAN(n-char-sequence)" that "the meaning of @@ -1197,7 +1158,7 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { if (strtoint_result.has_error()) { error = strtoint_result.error; } - nan_mantissa = static_cast(strtoint_result.value); + nan_mantissa = static_cast(strtoint_result.value); if (src[left_paren + 1 + strtoint_result.parsed_len] != ')') nan_mantissa = 0; } @@ -1207,11 +1168,11 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { } nan_mantissa |= fputil::FloatProperties::QUIET_NAN_MASK; if (result.get_sign()) { - result = fputil::FPBits(result.build_quiet_nan(nan_mantissa)); + result = FPBits(result.build_quiet_nan(nan_mantissa)); result.set_sign(true); } else { result.set_sign(false); - result = fputil::FPBits(result.build_quiet_nan(nan_mantissa)); + result = FPBits(result.build_quiet_nan(nan_mantissa)); } } } else if (tolower(src[index]) == 'i') { // INF @@ -1219,9 +1180,9 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { tolower(src[index + 2]) == inf_string[2]) { seen_digit = true; if (result.get_sign()) - result = fputil::FPBits(result.neg_inf()); + result = FPBits(result.neg_inf()); else - result = fputil::FPBits(result.inf()); + result = FPBits(result.inf()); if (tolower(src[index + 3]) == inf_string[3] && tolower(src[index + 4]) == inf_string[4] && tolower(src[index + 5]) == inf_string[5] && diff --git a/libc/src/math/generic/expf.cpp b/libc/src/math/generic/expf.cpp index 5a938a7a1c22e5e031b2c91e0a1c276db3c85cbc..12f62960fc10ff6fece88258323d83954480eeac 100644 --- a/libc/src/math/generic/expf.cpp +++ b/libc/src/math/generic/expf.cpp @@ -37,7 +37,7 @@ LLVM_LIBC_FUNCTION(float, expf, (float x)) { // When |x| >= 89, |x| < 2^-25, or x is nan if (LIBC_UNLIKELY(x_abs >= 0x42b2'0000U || x_abs <= 0x3280'0000U)) { // |x| < 2^-25 - if (xbits.get_unbiased_exponent() <= 101) { + if (xbits.get_biased_exponent() <= 101) { return 1.0f + x; } diff --git a/libc/src/math/generic/explogxf.h b/libc/src/math/generic/explogxf.h index 512785be2cb858b735661e69a2ac76c32fdd20dd..77ec9cb94e085455355f4a4e776516268e25e618 100644 --- a/libc/src/math/generic/explogxf.h +++ b/libc/src/math/generic/explogxf.h @@ -285,7 +285,7 @@ LIBC_INLINE static double log2_eval(double x) { (LOG_P1_SIZE - 1); bs.bits &= FPB::FloatProp::MANTISSA_MASK >> LOG_P1_BITS; - bs.set_unbiased_exponent(FPB::FloatProp::EXPONENT_BIAS); + bs.set_biased_exponent(FPB::FloatProp::EXPONENT_BIAS); double dx = (bs.get_val() - 1.0) * LOG_P1_1_OVER[p1]; // Taylor series for log(2,1+x) @@ -311,11 +311,12 @@ LIBC_INLINE static double log_eval(double x) { // p1 is the leading 7 bits of mx, i.e. // p1 * 2^(-7) <= m_x < (p1 + 1) * 2^(-7). - int p1 = (bs.get_mantissa() >> (FPB::FloatProp::MANTISSA_WIDTH - 7)); + int p1 = static_cast(bs.get_mantissa() >> + (FPB::FloatProp::MANTISSA_WIDTH - 7)); // Set bs to (1 + (mx - p1*2^(-7)) bs.bits &= FPB::FloatProp::MANTISSA_MASK >> 7; - bs.set_unbiased_exponent(FPB::FloatProp::EXPONENT_BIAS); + bs.set_biased_exponent(FPB::FloatProp::EXPONENT_BIAS); // dx = (mx - p1*2^(-7)) / (1 + p1*2^(-7)). double dx = (bs.get_val() - 1.0) * ONE_OVER_F[p1]; diff --git a/libc/src/math/generic/hypotf.cpp b/libc/src/math/generic/hypotf.cpp index 3fedeed4ed26b4e7a938ad908cc00d9b51236868..389de3c450299d479ad4c53a1daf8e9568b79ab3 100644 --- a/libc/src/math/generic/hypotf.cpp +++ b/libc/src/math/generic/hypotf.cpp @@ -19,8 +19,8 @@ LLVM_LIBC_FUNCTION(float, hypotf, (float x, float y)) { FPBits x_bits(x), y_bits(y); - uint16_t x_exp = x_bits.get_unbiased_exponent(); - uint16_t y_exp = y_bits.get_unbiased_exponent(); + uint16_t x_exp = x_bits.get_biased_exponent(); + uint16_t y_exp = y_bits.get_biased_exponent(); uint16_t exp_diff = (x_exp > y_exp) ? (x_exp - y_exp) : (y_exp - x_exp); if (exp_diff >= fputil::MantissaWidth::VALUE + 2) { diff --git a/libc/src/math/generic/log.cpp b/libc/src/math/generic/log.cpp index 46b64df689086d5423487ec27598b6b2ad64ff1c..dfa41ad64578d300e0c921a3752237bc51c1a24c 100644 --- a/libc/src/math/generic/log.cpp +++ b/libc/src/math/generic/log.cpp @@ -769,7 +769,7 @@ LLVM_LIBC_FUNCTION(double, log, (double x)) { // Range reduction for log(x_m): // For each x_m, we would like to find r such that: // -2^-8 <= r * x_m - 1 < 2^-7 - int shifted = x_u >> 45; + int shifted = static_cast(x_u >> 45); int index = shifted & 0x7F; double r = RD[index]; diff --git a/libc/src/math/generic/log10.cpp b/libc/src/math/generic/log10.cpp index 38789acc441e5b56e9e9e150b846af9a65f3265e..2a801c6e98429a686220fb55265403159cd56091 100644 --- a/libc/src/math/generic/log10.cpp +++ b/libc/src/math/generic/log10.cpp @@ -770,7 +770,7 @@ LLVM_LIBC_FUNCTION(double, log10, (double x)) { // Range reduction for log10(x_m): // For each x_m, we would like to find r such that: // -2^-8 <= r * x_m - 1 < 2^-7 - int shifted = x_u >> 45; + int shifted = static_cast(x_u >> 45); int index = shifted & 0x7F; double r = RD[index]; diff --git a/libc/src/math/generic/log10f.cpp b/libc/src/math/generic/log10f.cpp index bc9af75169b14b4ece9d0d6fac0ba355685a8067..927af35c98f28b0c9231a69f0d9b0781a6336353 100644 --- a/libc/src/math/generic/log10f.cpp +++ b/libc/src/math/generic/log10f.cpp @@ -187,7 +187,7 @@ LLVM_LIBC_FUNCTION(float, log10f, (float x)) { // Extract 7 leading fractional bits of the mantissa int index = (x_u >> 16) & 0x7F; // Set bits to 1.m - xbits.set_unbiased_exponent(0x7F); + xbits.set_biased_exponent(0x7F); float u = static_cast(xbits); double v; diff --git a/libc/src/math/generic/log1p.cpp b/libc/src/math/generic/log1p.cpp index c6ee8d8f9bbfbb8ef5557c8f8a030d5d7ca0a28b..c8b45fd57b42f8803bdd1102e7ff31085d6b89a1 100644 --- a/libc/src/math/generic/log1p.cpp +++ b/libc/src/math/generic/log1p.cpp @@ -880,7 +880,7 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) { fputil::DoubleDouble x_dd{0.0, 0.0}; - uint16_t x_exp = xbits.get_unbiased_exponent(); + uint16_t x_exp = xbits.get_biased_exponent(); if (x_exp >= EXPONENT_BIAS) { // |x| >= 1 @@ -909,7 +909,7 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) { } } else { // |x| < 1 - if (LIBC_UNLIKELY(xbits.get_unbiased_exponent() < + if (LIBC_UNLIKELY(xbits.get_biased_exponent() < EXPONENT_BIAS - MANTISSA_WIDTH - 1)) { // Quick return when |x| < 2^-53. // Since log(1 + x) = x - x^2/2 + x^3/3 - ..., @@ -949,8 +949,9 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) { x_u = xhi_bits.uintval(); // Range reduction: // Find k such that |x_hi - k * 2^-7| <= 2^-8. - int idx = ((x_u & MANTISSA_MASK) + (1ULL << (MANTISSA_WIDTH - 8))) >> - (MANTISSA_WIDTH - 7); + int idx = static_cast( + ((x_u & MANTISSA_MASK) + (1ULL << (MANTISSA_WIDTH - 8))) >> + (MANTISSA_WIDTH - 7)); int x_e = xhi_bits.get_exponent() + (idx >> 7); double e_x = static_cast(x_e); diff --git a/libc/src/math/generic/log1pf.cpp b/libc/src/math/generic/log1pf.cpp index 5b4e7edcceb47271aa4ce84ef430887360d801d3..fd3cf4647fd0da93f3c9bb5330b8118eb2f6047e 100644 --- a/libc/src/math/generic/log1pf.cpp +++ b/libc/src/math/generic/log1pf.cpp @@ -56,11 +56,11 @@ LIBC_INLINE float log(double x) { // Get the 8 highest bits, use 7 bits (excluding the implicit hidden bit) for // lookup tables. - int f_index = - xbits.get_mantissa() >> 45; // fputil::MantissaWidth::VALUE - 7 + int f_index = static_cast( + xbits.get_mantissa() >> 45); // fputil::MantissaWidth::VALUE - 7 // Set bits to 1.m - xbits.set_unbiased_exponent(0x3FF); + xbits.set_biased_exponent(0x3FF); FPBits f = xbits; // Clear the lowest 45 bits. diff --git a/libc/src/math/generic/log2.cpp b/libc/src/math/generic/log2.cpp index d72b0931c14b81b678184c0a4e9fcb7e83712b49..2ceddf87dfd56090e31b41aeb0bf35787e7f4c6e 100644 --- a/libc/src/math/generic/log2.cpp +++ b/libc/src/math/generic/log2.cpp @@ -890,7 +890,7 @@ LLVM_LIBC_FUNCTION(double, log2, (double x)) { // Range reduction for log2(x_m): // For each x_m, we would like to find r such that: // -2^-8 <= r * x_m - 1 < 2^-7 - int shifted = x_u >> 45; + int shifted = static_cast(x_u >> 45); int index = shifted & 0x7F; double r = RD[index]; diff --git a/libc/src/math/generic/log2f.cpp b/libc/src/math/generic/log2f.cpp index 7665a90f092359fb74e918929aa75be6b5679c80..4bbc9f51c3d2af7c0736afd91102bb45cd312deb 100644 --- a/libc/src/math/generic/log2f.cpp +++ b/libc/src/math/generic/log2f.cpp @@ -87,10 +87,10 @@ LLVM_LIBC_FUNCTION(float, log2f, (float x)) { m -= 23; } - m += xbits.get_unbiased_exponent(); + m += xbits.get_biased_exponent(); int index = xbits.get_mantissa() >> 16; // Set bits to 1.m - xbits.set_unbiased_exponent(0x7F); + xbits.set_biased_exponent(0x7F); float u = static_cast(xbits); double v; diff --git a/libc/src/math/generic/logf.cpp b/libc/src/math/generic/logf.cpp index 1f689f25931d4e8a464906847779a29f007758e9..2dfada38b972383bcfa68bc907b3d9f46a9762bd 100644 --- a/libc/src/math/generic/logf.cpp +++ b/libc/src/math/generic/logf.cpp @@ -135,7 +135,7 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) { // rounding mode. if (LIBC_UNLIKELY((x_u & 0x007f'ffffU) == 0)) return static_cast( - static_cast(m + xbits.get_unbiased_exponent()) * LOG_2); + static_cast(m + xbits.get_biased_exponent()) * LOG_2); #endif // LIBC_TARGET_CPU_HAS_FMA uint32_t mant = xbits.get_mantissa(); @@ -146,7 +146,7 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) { m += static_cast((x_u + (1 << 16)) >> 23); // Set bits to 1.m - xbits.set_unbiased_exponent(0x7F); + xbits.set_biased_exponent(0x7F); float u = static_cast(xbits); double v; diff --git a/libc/src/math/generic/powf.cpp b/libc/src/math/generic/powf.cpp index 5f2e95b44e5287991809a881746b8cfc517862b7..17ccb41fc462aad65d365622c074e9fead58581c 100644 --- a/libc/src/math/generic/powf.cpp +++ b/libc/src/math/generic/powf.cpp @@ -410,8 +410,8 @@ LIBC_INLINE bool is_integer(float x) { LIBC_INLINE bool larger_exponent(double a, double b) { using DoubleBits = typename fputil::FPBits; - return DoubleBits(a).get_unbiased_exponent() >= - DoubleBits(b).get_unbiased_exponent(); + return DoubleBits(a).get_biased_exponent() >= + DoubleBits(b).get_biased_exponent(); } // Calculate 2^(y * log2(x)) in double-double precision. diff --git a/libc/test/src/__support/FPUtil/fpbits_test.cpp b/libc/test/src/__support/FPUtil/fpbits_test.cpp index 52635cc2af0940186a9d62246c6d1e060ab52bc5..fa743855c4861963c64a79529126741b1a9efc11 100644 --- a/libc/test/src/__support/FPUtil/fpbits_test.cpp +++ b/libc/test/src/__support/FPUtil/fpbits_test.cpp @@ -24,7 +24,7 @@ TEST(LlvmLibcFPBitsTest, FloatType) { FloatBits zero(0.0f); EXPECT_EQ(zero.get_sign(), false); - EXPECT_EQ(zero.get_unbiased_exponent(), static_cast(0)); + EXPECT_EQ(zero.get_biased_exponent(), static_cast(0)); EXPECT_EQ(zero.get_mantissa(), static_cast(0)); EXPECT_EQ(zero.uintval(), static_cast(0x00000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(zero).c_str(), @@ -32,7 +32,7 @@ TEST(LlvmLibcFPBitsTest, FloatType) { FloatBits negzero(-0.0f); EXPECT_EQ(negzero.get_sign(), true); - EXPECT_EQ(negzero.get_unbiased_exponent(), static_cast(0)); + EXPECT_EQ(negzero.get_biased_exponent(), static_cast(0)); EXPECT_EQ(negzero.get_mantissa(), static_cast(0)); EXPECT_EQ(negzero.uintval(), static_cast(0x80000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(negzero).c_str(), @@ -40,7 +40,7 @@ TEST(LlvmLibcFPBitsTest, FloatType) { FloatBits one(1.0f); EXPECT_EQ(one.get_sign(), false); - EXPECT_EQ(one.get_unbiased_exponent(), static_cast(0x7F)); + EXPECT_EQ(one.get_biased_exponent(), static_cast(0x7F)); EXPECT_EQ(one.get_mantissa(), static_cast(0)); EXPECT_EQ(one.uintval(), static_cast(0x3F800000)); EXPECT_STREQ(LIBC_NAMESPACE::str(one).c_str(), @@ -48,7 +48,7 @@ TEST(LlvmLibcFPBitsTest, FloatType) { FloatBits negone(-1.0f); EXPECT_EQ(negone.get_sign(), true); - EXPECT_EQ(negone.get_unbiased_exponent(), static_cast(0x7F)); + EXPECT_EQ(negone.get_biased_exponent(), static_cast(0x7F)); EXPECT_EQ(negone.get_mantissa(), static_cast(0)); EXPECT_EQ(negone.uintval(), static_cast(0xBF800000)); EXPECT_STREQ(LIBC_NAMESPACE::str(negone).c_str(), @@ -56,7 +56,7 @@ TEST(LlvmLibcFPBitsTest, FloatType) { FloatBits num(1.125f); EXPECT_EQ(num.get_sign(), false); - EXPECT_EQ(num.get_unbiased_exponent(), static_cast(0x7F)); + EXPECT_EQ(num.get_biased_exponent(), static_cast(0x7F)); EXPECT_EQ(num.get_mantissa(), static_cast(0x00100000)); EXPECT_EQ(num.uintval(), static_cast(0x3F900000)); EXPECT_STREQ(LIBC_NAMESPACE::str(num).c_str(), @@ -64,7 +64,7 @@ TEST(LlvmLibcFPBitsTest, FloatType) { FloatBits negnum(-1.125f); EXPECT_EQ(negnum.get_sign(), true); - EXPECT_EQ(negnum.get_unbiased_exponent(), static_cast(0x7F)); + EXPECT_EQ(negnum.get_biased_exponent(), static_cast(0x7F)); EXPECT_EQ(negnum.get_mantissa(), static_cast(0x00100000)); EXPECT_EQ(negnum.uintval(), static_cast(0xBF900000)); EXPECT_STREQ(LIBC_NAMESPACE::str(negnum).c_str(), @@ -84,7 +84,7 @@ TEST(LlvmLibcFPBitsTest, DoubleType) { DoubleBits zero(0.0); EXPECT_EQ(zero.get_sign(), false); - EXPECT_EQ(zero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(zero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(zero.get_mantissa(), static_cast(0x0000000000000000)); EXPECT_EQ(zero.uintval(), static_cast(0x0000000000000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(zero).c_str(), @@ -92,7 +92,7 @@ TEST(LlvmLibcFPBitsTest, DoubleType) { DoubleBits negzero(-0.0); EXPECT_EQ(negzero.get_sign(), true); - EXPECT_EQ(negzero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(negzero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(negzero.get_mantissa(), static_cast(0x0000000000000000)); EXPECT_EQ(negzero.uintval(), static_cast(0x8000000000000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(negzero).c_str(), @@ -100,7 +100,7 @@ TEST(LlvmLibcFPBitsTest, DoubleType) { DoubleBits one(1.0); EXPECT_EQ(one.get_sign(), false); - EXPECT_EQ(one.get_unbiased_exponent(), static_cast(0x03FF)); + EXPECT_EQ(one.get_biased_exponent(), static_cast(0x03FF)); EXPECT_EQ(one.get_mantissa(), static_cast(0x0000000000000000)); EXPECT_EQ(one.uintval(), static_cast(0x3FF0000000000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(one).c_str(), @@ -108,7 +108,7 @@ TEST(LlvmLibcFPBitsTest, DoubleType) { DoubleBits negone(-1.0); EXPECT_EQ(negone.get_sign(), true); - EXPECT_EQ(negone.get_unbiased_exponent(), static_cast(0x03FF)); + EXPECT_EQ(negone.get_biased_exponent(), static_cast(0x03FF)); EXPECT_EQ(negone.get_mantissa(), static_cast(0x0000000000000000)); EXPECT_EQ(negone.uintval(), static_cast(0xBFF0000000000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(negone).c_str(), @@ -116,7 +116,7 @@ TEST(LlvmLibcFPBitsTest, DoubleType) { DoubleBits num(1.125); EXPECT_EQ(num.get_sign(), false); - EXPECT_EQ(num.get_unbiased_exponent(), static_cast(0x03FF)); + EXPECT_EQ(num.get_biased_exponent(), static_cast(0x03FF)); EXPECT_EQ(num.get_mantissa(), static_cast(0x0002000000000000)); EXPECT_EQ(num.uintval(), static_cast(0x3FF2000000000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(num).c_str(), @@ -124,7 +124,7 @@ TEST(LlvmLibcFPBitsTest, DoubleType) { DoubleBits negnum(-1.125); EXPECT_EQ(negnum.get_sign(), true); - EXPECT_EQ(negnum.get_unbiased_exponent(), static_cast(0x03FF)); + EXPECT_EQ(negnum.get_biased_exponent(), static_cast(0x03FF)); EXPECT_EQ(negnum.get_mantissa(), static_cast(0x0002000000000000)); EXPECT_EQ(negnum.uintval(), static_cast(0xBFF2000000000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(negnum).c_str(), @@ -150,7 +150,7 @@ TEST(LlvmLibcFPBitsTest, X86LongDoubleType) { LongDoubleBits zero(0.0l); EXPECT_EQ(zero.get_sign(), false); - EXPECT_EQ(zero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(zero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(zero.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(zero.uintval(), static_cast(0x0000000000000000) << 64); @@ -161,7 +161,7 @@ TEST(LlvmLibcFPBitsTest, X86LongDoubleType) { LongDoubleBits negzero(-0.0l); EXPECT_EQ(negzero.get_sign(), true); - EXPECT_EQ(negzero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(negzero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(negzero.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(negzero.uintval(), static_cast(0x1) << 79); @@ -172,7 +172,7 @@ TEST(LlvmLibcFPBitsTest, X86LongDoubleType) { LongDoubleBits one(1.0l); EXPECT_EQ(one.get_sign(), false); - EXPECT_EQ(one.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(one.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(one.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(one.uintval(), static_cast(0x3FFF8) << 60); EXPECT_STREQ( @@ -182,7 +182,7 @@ TEST(LlvmLibcFPBitsTest, X86LongDoubleType) { LongDoubleBits negone(-1.0l); EXPECT_EQ(negone.get_sign(), true); - EXPECT_EQ(negone.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(negone.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(negone.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(negone.uintval(), static_cast(0xBFFF8) << 60); @@ -193,7 +193,7 @@ TEST(LlvmLibcFPBitsTest, X86LongDoubleType) { LongDoubleBits num(1.125l); EXPECT_EQ(num.get_sign(), false); - EXPECT_EQ(num.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(num.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(num.get_mantissa(), static_cast(0x1) << 60); EXPECT_EQ(num.uintval(), static_cast(0x3FFF9) << 60); EXPECT_STREQ( @@ -203,7 +203,7 @@ TEST(LlvmLibcFPBitsTest, X86LongDoubleType) { LongDoubleBits negnum(-1.125l); EXPECT_EQ(negnum.get_sign(), true); - EXPECT_EQ(negnum.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(negnum.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(negnum.get_mantissa(), static_cast(0x1) << 60); EXPECT_EQ(negnum.uintval(), static_cast(0xBFFF9) << 60); EXPECT_STREQ( @@ -230,7 +230,7 @@ TEST(LlvmLibcFPBitsTest, LongDoubleType) { LongDoubleBits zero(0.0l); EXPECT_EQ(zero.get_sign(), false); - EXPECT_EQ(zero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(zero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(zero.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(zero.uintval(), static_cast(0x0000000000000000) << 64); @@ -240,7 +240,7 @@ TEST(LlvmLibcFPBitsTest, LongDoubleType) { LongDoubleBits negzero(-0.0l); EXPECT_EQ(negzero.get_sign(), true); - EXPECT_EQ(negzero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(negzero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(negzero.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(negzero.uintval(), static_cast(0x1) << 127); @@ -250,7 +250,7 @@ TEST(LlvmLibcFPBitsTest, LongDoubleType) { LongDoubleBits one(1.0l); EXPECT_EQ(one.get_sign(), false); - EXPECT_EQ(one.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(one.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(one.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(one.uintval(), static_cast(0x3FFF) << 112); EXPECT_STREQ(LIBC_NAMESPACE::str(one).c_str(), @@ -259,7 +259,7 @@ TEST(LlvmLibcFPBitsTest, LongDoubleType) { LongDoubleBits negone(-1.0l); EXPECT_EQ(negone.get_sign(), true); - EXPECT_EQ(negone.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(negone.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(negone.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(negone.uintval(), static_cast(0xBFFF) << 112); @@ -269,7 +269,7 @@ TEST(LlvmLibcFPBitsTest, LongDoubleType) { LongDoubleBits num(1.125l); EXPECT_EQ(num.get_sign(), false); - EXPECT_EQ(num.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(num.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(num.get_mantissa(), static_cast(0x2) << 108); EXPECT_EQ(num.uintval(), static_cast(0x3FFF2) << 108); EXPECT_STREQ(LIBC_NAMESPACE::str(num).c_str(), @@ -278,7 +278,7 @@ TEST(LlvmLibcFPBitsTest, LongDoubleType) { LongDoubleBits negnum(-1.125l); EXPECT_EQ(negnum.get_sign(), true); - EXPECT_EQ(negnum.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(negnum.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(negnum.get_mantissa(), static_cast(0x2) << 108); EXPECT_EQ(negnum.uintval(), static_cast(0xBFFF2) << 108); EXPECT_STREQ(LIBC_NAMESPACE::str(negnum).c_str(), @@ -303,7 +303,7 @@ TEST(LlvmLibcFPBitsTest, Float128Type) { Float128Bits zero(Float128Bits::zero()); EXPECT_EQ(zero.get_sign(), false); - EXPECT_EQ(zero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(zero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(zero.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(zero.uintval(), static_cast(0x0000000000000000) << 64); @@ -313,7 +313,7 @@ TEST(LlvmLibcFPBitsTest, Float128Type) { Float128Bits negzero(Float128Bits::neg_zero()); EXPECT_EQ(negzero.get_sign(), true); - EXPECT_EQ(negzero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(negzero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(negzero.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(negzero.uintval(), static_cast(0x1) << 127); @@ -323,7 +323,7 @@ TEST(LlvmLibcFPBitsTest, Float128Type) { Float128Bits one(float128(1.0)); EXPECT_EQ(one.get_sign(), false); - EXPECT_EQ(one.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(one.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(one.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(one.uintval(), static_cast(0x3FFF) << 112); EXPECT_STREQ(LIBC_NAMESPACE::str(one).c_str(), @@ -332,7 +332,7 @@ TEST(LlvmLibcFPBitsTest, Float128Type) { Float128Bits negone(float128(-1.0)); EXPECT_EQ(negone.get_sign(), true); - EXPECT_EQ(negone.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(negone.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(negone.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(negone.uintval(), static_cast(0xBFFF) << 112); @@ -342,7 +342,7 @@ TEST(LlvmLibcFPBitsTest, Float128Type) { Float128Bits num(float128(1.125)); EXPECT_EQ(num.get_sign(), false); - EXPECT_EQ(num.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(num.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(num.get_mantissa(), static_cast(0x2) << 108); EXPECT_EQ(num.uintval(), static_cast(0x3FFF2) << 108); EXPECT_STREQ(LIBC_NAMESPACE::str(num).c_str(), @@ -351,7 +351,7 @@ TEST(LlvmLibcFPBitsTest, Float128Type) { Float128Bits negnum(float128(-1.125)); EXPECT_EQ(negnum.get_sign(), true); - EXPECT_EQ(negnum.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(negnum.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(negnum.get_mantissa(), static_cast(0x2) << 108); EXPECT_EQ(negnum.uintval(), static_cast(0xBFFF2) << 108); EXPECT_STREQ(LIBC_NAMESPACE::str(negnum).c_str(), diff --git a/libc/test/src/__support/str_to_float_test.cpp b/libc/test/src/__support/str_to_float_test.cpp index f9d12d95a50beeb86175f6ce4ba577d5c2042474..35f7318fb9c78df71f1fb3f591533f5d20b63e51 100644 --- a/libc/test/src/__support/str_to_float_test.cpp +++ b/libc/test/src/__support/str_to_float_test.cpp @@ -93,45 +93,6 @@ public: } }; -TEST(LlvmLibcStrToFloatTest, LeadingZeroes) { - uint64_t test_num64 = 1; - uint32_t num_of_zeroes = 63; - EXPECT_EQ(LIBC_NAMESPACE::internal::leading_zeroes(0), 64u); - for (; num_of_zeroes < 64; test_num64 <<= 1, num_of_zeroes--) { - EXPECT_EQ(LIBC_NAMESPACE::internal::leading_zeroes(test_num64), - num_of_zeroes); - } - - test_num64 = 3; - num_of_zeroes = 62; - for (; num_of_zeroes > 63; test_num64 <<= 1, num_of_zeroes--) { - EXPECT_EQ(LIBC_NAMESPACE::internal::leading_zeroes(test_num64), - num_of_zeroes); - } - - EXPECT_EQ( - LIBC_NAMESPACE::internal::leading_zeroes(0xffffffffffffffff), - 0u); - - test_num64 = 1; - num_of_zeroes = 63; - for (; num_of_zeroes > 63; - test_num64 = (test_num64 << 1) + 1, num_of_zeroes--) { - EXPECT_EQ(LIBC_NAMESPACE::internal::leading_zeroes(test_num64), - num_of_zeroes); - } - - uint64_t test_num32 = 1; - num_of_zeroes = 31; - EXPECT_EQ(LIBC_NAMESPACE::internal::leading_zeroes(0), 32u); - for (; num_of_zeroes < 32; test_num32 <<= 1, num_of_zeroes--) { - EXPECT_EQ(LIBC_NAMESPACE::internal::leading_zeroes(test_num32), - num_of_zeroes); - } - - EXPECT_EQ(LIBC_NAMESPACE::internal::leading_zeroes(0xffffffff), 0u); -} - TEST_F(LlvmLibcStrToFloatTest, ClingerFastPathFloat64Simple) { clinger_fast_path_test(123, 0, 0xEC00000000000, 1029); clinger_fast_path_test(1234567890123456, 1, 0x5ee2a2eb5a5c0, 1076); diff --git a/libc/test/src/math/LdExpTest.h b/libc/test/src/math/LdExpTest.h index bffa7335fb25b17f206535adc602e83401dfc265..a75c8ef31a2cf83e9e279008eff5ec7553a0919e 100644 --- a/libc/test/src/math/LdExpTest.h +++ b/libc/test/src/math/LdExpTest.h @@ -132,7 +132,7 @@ public: FPBits result_bits(result); ASSERT_FALSE(result_bits.is_zero()); // Verify that the result is indeed subnormal. - ASSERT_EQ(result_bits.get_unbiased_exponent(), uint16_t(0)); + ASSERT_EQ(result_bits.get_biased_exponent(), uint16_t(0)); // But if the exp is so less that normalization leads to zero, then // the result should be zero. result = func(x, -FPBits::MAX_EXPONENT - int(MANTISSA_WIDTH) - 5); diff --git a/libc/test/src/math/NextAfterTest.h b/libc/test/src/math/NextAfterTest.h index 4e450cf5bc1e9e79cd97709b29c68b5b0596f761..57a801dfb28a316ae9f0a2d623217f199f2a3359 100644 --- a/libc/test/src/math/NextAfterTest.h +++ b/libc/test/src/math/NextAfterTest.h @@ -162,30 +162,28 @@ public: result = func(x, 0); FPBits x_bits = FPBits(x); FPBits result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - uint16_t(x_bits.get_unbiased_exponent() - 1)); + ASSERT_EQ(result_bits.get_biased_exponent(), + uint16_t(x_bits.get_biased_exponent() - 1)); ASSERT_EQ(result_bits.get_mantissa(), (UIntType(1) << MantissaWidth::VALUE) - 1); result = func(x, T(33.0)); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - x_bits.get_unbiased_exponent()); + ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), x_bits.get_mantissa() + UIntType(1)); x = -x; result = func(x, 0); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - uint16_t(x_bits.get_unbiased_exponent() - 1)); + ASSERT_EQ(result_bits.get_biased_exponent(), + uint16_t(x_bits.get_biased_exponent() - 1)); ASSERT_EQ(result_bits.get_mantissa(), (UIntType(1) << MantissaWidth::VALUE) - 1); result = func(x, T(-33.0)); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - x_bits.get_unbiased_exponent()); + ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), x_bits.get_mantissa() + UIntType(1)); } }; diff --git a/libc/test/src/math/RoundToIntegerTest.h b/libc/test/src/math/RoundToIntegerTest.h index 6b205869e7a7a7a9545614e2f2023ddb9bc4b41a..1a976e97359eb76ab108a1a7ca824767c851946c 100644 --- a/libc/test/src/math/RoundToIntegerTest.h +++ b/libc/test/src/math/RoundToIntegerTest.h @@ -126,7 +126,7 @@ public: // We start with 1.0 so that the implicit bit for x86 long doubles // is set. FPBits bits(F(1.0)); - bits.set_unbiased_exponent(EXPONENT_LIMIT + FPBits::EXPONENT_BIAS); + bits.set_biased_exponent(EXPONENT_LIMIT + FPBits::EXPONENT_BIAS); bits.set_sign(1); bits.set_mantissa(0); @@ -190,7 +190,7 @@ public: // We start with 1.0 so that the implicit bit for x86 long doubles // is set. FPBits bits(F(1.0)); - bits.set_unbiased_exponent(EXPONENT_LIMIT + FPBits::EXPONENT_BIAS); + bits.set_biased_exponent(EXPONENT_LIMIT + FPBits::EXPONENT_BIAS); bits.set_sign(1); bits.set_mantissa(UIntType(0x1) << (LIBC_NAMESPACE::fputil::MantissaWidth::VALUE - 1)); diff --git a/libc/test/src/math/smoke/LdExpTest.h b/libc/test/src/math/smoke/LdExpTest.h index bffa7335fb25b17f206535adc602e83401dfc265..a75c8ef31a2cf83e9e279008eff5ec7553a0919e 100644 --- a/libc/test/src/math/smoke/LdExpTest.h +++ b/libc/test/src/math/smoke/LdExpTest.h @@ -132,7 +132,7 @@ public: FPBits result_bits(result); ASSERT_FALSE(result_bits.is_zero()); // Verify that the result is indeed subnormal. - ASSERT_EQ(result_bits.get_unbiased_exponent(), uint16_t(0)); + ASSERT_EQ(result_bits.get_biased_exponent(), uint16_t(0)); // But if the exp is so less that normalization leads to zero, then // the result should be zero. result = func(x, -FPBits::MAX_EXPONENT - int(MANTISSA_WIDTH) - 5); diff --git a/libc/test/src/math/smoke/NextAfterTest.h b/libc/test/src/math/smoke/NextAfterTest.h index 1b082050a598c3ee6546608539c94e0ef3db9714..29098e0f49a45755d9000fc12dcddecd3f6b6a1d 100644 --- a/libc/test/src/math/smoke/NextAfterTest.h +++ b/libc/test/src/math/smoke/NextAfterTest.h @@ -173,30 +173,28 @@ public: result = func(x, 0); FPBits x_bits = FPBits(x); FPBits result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - uint16_t(x_bits.get_unbiased_exponent() - 1)); + ASSERT_EQ(result_bits.get_biased_exponent(), + uint16_t(x_bits.get_biased_exponent() - 1)); ASSERT_EQ(result_bits.get_mantissa(), (UIntType(1) << MantissaWidth::VALUE) - 1); result = func(x, T(33.0)); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - x_bits.get_unbiased_exponent()); + ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), x_bits.get_mantissa() + UIntType(1)); x = -x; result = func(x, 0); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - uint16_t(x_bits.get_unbiased_exponent() - 1)); + ASSERT_EQ(result_bits.get_biased_exponent(), + uint16_t(x_bits.get_biased_exponent() - 1)); ASSERT_EQ(result_bits.get_mantissa(), (UIntType(1) << MantissaWidth::VALUE) - 1); result = func(x, T(-33.0)); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - x_bits.get_unbiased_exponent()); + ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), x_bits.get_mantissa() + UIntType(1)); } }; diff --git a/libc/test/src/math/smoke/NextTowardTest.h b/libc/test/src/math/smoke/NextTowardTest.h index 4d27592a11422bfe4e26d8d8e2f6a19a2fedf39a..111d8017e691d36841de58843b062be46f0e0c4b 100644 --- a/libc/test/src/math/smoke/NextTowardTest.h +++ b/libc/test/src/math/smoke/NextTowardTest.h @@ -187,30 +187,28 @@ public: result = func(x, 0); FPBits x_bits = FPBits(x); FPBits result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - uint16_t(x_bits.get_unbiased_exponent() - 1)); + ASSERT_EQ(result_bits.get_biased_exponent(), + uint16_t(x_bits.get_biased_exponent() - 1)); ASSERT_EQ(result_bits.get_mantissa(), (UIntType(1) << MantissaWidth::VALUE) - 1); result = func(x, 33.0); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - x_bits.get_unbiased_exponent()); + ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), x_bits.get_mantissa() + UIntType(1)); x = -x; result = func(x, 0); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - uint16_t(x_bits.get_unbiased_exponent() - 1)); + ASSERT_EQ(result_bits.get_biased_exponent(), + uint16_t(x_bits.get_biased_exponent() - 1)); ASSERT_EQ(result_bits.get_mantissa(), (UIntType(1) << MantissaWidth::VALUE) - 1); result = func(x, -33.0); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - x_bits.get_unbiased_exponent()); + ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), x_bits.get_mantissa() + UIntType(1)); } }; diff --git a/libc/test/utils/FPUtil/x86_long_double_test.cpp b/libc/test/utils/FPUtil/x86_long_double_test.cpp index 6ca9b16377c0b80efc58f440975753c0470034b5..cea43c1a6fa4d5a60da1fcfc46e48a918233c24e 100644 --- a/libc/test/utils/FPUtil/x86_long_double_test.cpp +++ b/libc/test/utils/FPUtil/x86_long_double_test.cpp @@ -22,7 +22,7 @@ TEST(LlvmLibcX86LongDoubleTest, is_nan) { constexpr uint32_t COUNT = 100'000; FPBits bits(0.0l); - bits.set_unbiased_exponent(FPBits::MAX_EXPONENT); + bits.set_biased_exponent(FPBits::MAX_EXPONENT); for (unsigned int i = 0; i < COUNT; ++i) { // If exponent has the max value and the implicit bit is 0, // then the number is a NaN for all values of mantissa. @@ -43,7 +43,7 @@ TEST(LlvmLibcX86LongDoubleTest, is_nan) { ASSERT_TRUE(bits.is_nan()); } - bits.set_unbiased_exponent(1); + bits.set_biased_exponent(1); bits.set_implicit_bit(0); for (unsigned int i = 0; i < COUNT; ++i) { // If exponent is non-zero and also not max, and the implicit bit is 0, @@ -54,7 +54,7 @@ TEST(LlvmLibcX86LongDoubleTest, is_nan) { ASSERT_TRUE(bits.is_nan()); } - bits.set_unbiased_exponent(1); + bits.set_biased_exponent(1); bits.set_implicit_bit(1); for (unsigned int i = 0; i < COUNT; ++i) { // If exponent is non-zero and also not max, and the implicit bit is 1, @@ -65,7 +65,7 @@ TEST(LlvmLibcX86LongDoubleTest, is_nan) { ASSERT_FALSE(bits.is_nan()); } - bits.set_unbiased_exponent(0); + bits.set_biased_exponent(0); bits.set_implicit_bit(1); for (unsigned int i = 0; i < COUNT; ++i) { // If exponent is zero, then the number is a valid but denormal value. @@ -75,7 +75,7 @@ TEST(LlvmLibcX86LongDoubleTest, is_nan) { ASSERT_FALSE(bits.is_nan()); } - bits.set_unbiased_exponent(0); + bits.set_biased_exponent(0); bits.set_implicit_bit(0); for (unsigned int i = 0; i < COUNT; ++i) { // If exponent is zero, then the number is a valid but denormal value. diff --git a/libc/utils/MPFRWrapper/MPFRUtils.cpp b/libc/utils/MPFRWrapper/MPFRUtils.cpp index e1213e00ff361407a33a192af1828590955c454d..e3dffadf4ed9674fae495f71cebf6927d4366dc0 100644 --- a/libc/utils/MPFRWrapper/MPFRUtils.cpp +++ b/libc/utils/MPFRWrapper/MPFRUtils.cpp @@ -458,9 +458,9 @@ public: int thisExponent = fputil::FPBits(thisAsT).get_exponent(); int inputExponent = fputil::FPBits(input).get_exponent(); // Adjust the exponents for denormal numbers. - if (fputil::FPBits(thisAsT).get_unbiased_exponent() == 0) + if (fputil::FPBits(thisAsT).get_biased_exponent() == 0) ++thisExponent; - if (fputil::FPBits(input).get_unbiased_exponent() == 0) + if (fputil::FPBits(input).get_biased_exponent() == 0) ++inputExponent; if (thisAsT * input < 0 || thisExponent == inputExponent) { @@ -483,9 +483,9 @@ public: int minExponent = fputil::FPBits(min).get_exponent(); int maxExponent = fputil::FPBits(max).get_exponent(); // Adjust the exponents for denormal numbers. - if (fputil::FPBits(min).get_unbiased_exponent() == 0) + if (fputil::FPBits(min).get_biased_exponent() == 0) ++minExponent; - if (fputil::FPBits(max).get_unbiased_exponent() == 0) + if (fputil::FPBits(max).get_biased_exponent() == 0) ++maxExponent; MPFRNumber minMPFR(min); diff --git a/libcxx/docs/ReleaseNotes/18.rst b/libcxx/docs/ReleaseNotes/18.rst index abefe4c28ca95807f58c0ea525f268b00339b55f..9e509db6359c4aa7fdbc9ff25e8b7a64a0235653 100644 --- a/libcxx/docs/ReleaseNotes/18.rst +++ b/libcxx/docs/ReleaseNotes/18.rst @@ -50,6 +50,7 @@ Implemented Papers - P0053R7 - C++ Synchronized Buffered Ostream (in the experimental library) - P2467R1 - Support exclusive mode for fstreams - P0020R6 - Floating Point Atomic +- P2905R2 - Runtime format strings - P2918R2 - Runtime format strings II - P2871R3 - Remove Deprecated Unicode Conversion Facets from C++26 - P2870R3 - Remove basic_string::reserve() diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv index 1d071b7ebcb4a76fcae71480eb83148a17e4191d..ff83648aa76830a72ddc4fc68889e032b309edb9 100644 --- a/libcxx/docs/Status/Cxx2cPapers.csv +++ b/libcxx/docs/Status/Cxx2cPapers.csv @@ -30,7 +30,7 @@ "`P0543R3 `__","LWG","Saturation arithmetic","Kona November 2023","","","" "`P2407R5 `__","LWG","Freestanding Library: Partial Classes","Kona November 2023","","","" "`P2546R5 `__","LWG","Debugging Support","Kona November 2023","","","" -"`P2905R2 `__","LWG","Runtime format strings","Kona November 2023","","","|format| |DR|" +"`P2905R2 `__","LWG","Runtime format strings","Kona November 2023","|Complete|","18.0","|format| |DR|" "`P2918R2 `__","LWG","Runtime format strings II","Kona November 2023","|Complete|","18.0","|format|" "`P2909R4 `__","LWG","Fix formatting of code units as integers (Dude, where’s my ``char``?)","Kona November 2023","|Complete|","18.0","|format| |DR|" "`P0952R2 `__","LWG","A new specification for ``std::generate_canonical``","Kona November 2023","","","" diff --git a/libcxx/docs/Status/FormatIssues.csv b/libcxx/docs/Status/FormatIssues.csv index 005de97405f7ccda421674cd92d03fdf8f96539c..efb3e484f357e72117d6bc7097403a20e9d368a2 100644 --- a/libcxx/docs/Status/FormatIssues.csv +++ b/libcxx/docs/Status/FormatIssues.csv @@ -17,7 +17,7 @@ Number,Name,Standard,Assignee,Status,First released version "`P2510R3 `__","Formatting pointers","C++26","Mark de Wever","|Complete|",17.0 "`P2757R3 `__","Type-checking format args","C++26","","", "`P2637R3 `__","Member ``visit``","C++26","","", -"`P2905R2 `__","Runtime format strings","C++26 DR","Mark de Wever","|In Progress|" +"`P2905R2 `__","Runtime format strings","C++26 DR","Mark de Wever","|Complete|",18.0 "`P2918R2 `__","Runtime format strings II","C++26","Mark de Wever","|Complete|",18.0 "`P2909R4 `__","Fix formatting of code units as integers (Dude, where’s my ``char``?)","C++26 DR","Mark de Wever","|Complete|",18.0 `P1361 `_,"Integration of chrono with text formatting","C++20",Mark de Wever,|In Progress|, diff --git a/libcxx/include/__format/format_arg_store.h b/libcxx/include/__format/format_arg_store.h index 2962962ab5d1c23c37471ec2ccfefe50b44fe944..64ee12440b62f8f578ce30c055625ceae8c13587 100644 --- a/libcxx/include/__format/format_arg_store.h +++ b/libcxx/include/__format/format_arg_store.h @@ -206,8 +206,8 @@ _LIBCPP_HIDE_FROM_ABI basic_format_arg<_Context> __create_format_arg(_Tp& __valu } template -_LIBCPP_HIDE_FROM_ABI void __create_packed_storage(uint64_t& __types, __basic_format_arg_value<_Context>* __values, - _Args&&... __args) noexcept { +_LIBCPP_HIDE_FROM_ABI void +__create_packed_storage(uint64_t& __types, __basic_format_arg_value<_Context>* __values, _Args&... __args) noexcept { int __shift = 0; ( [&] { @@ -224,7 +224,7 @@ _LIBCPP_HIDE_FROM_ABI void __create_packed_storage(uint64_t& __types, __basic_fo } template -_LIBCPP_HIDE_FROM_ABI void __store_basic_format_arg(basic_format_arg<_Context>* __data, _Args&&... __args) noexcept { +_LIBCPP_HIDE_FROM_ABI void __store_basic_format_arg(basic_format_arg<_Context>* __data, _Args&... __args) noexcept { ([&] { *__data++ = __format::__create_format_arg<_Context>(__args); }(), ...); } diff --git a/libcxx/include/__format/format_functions.h b/libcxx/include/__format/format_functions.h index 164592d2ec152355fc9417b17994077bb4b9dfc2..8b2111f0e287cbd69117223d355a1f0a6e762746 100644 --- a/libcxx/include/__format/format_functions.h +++ b/libcxx/include/__format/format_functions.h @@ -63,15 +63,15 @@ using wformat_args = basic_format_args; #endif template -_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI __format_arg_store<_Context, _Args...> make_format_args(_Args&&... __args) { - return std::__format_arg_store<_Context, _Args...>(__args...); +_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI __format_arg_store<_Context, _Args...> make_format_args(_Args&... __args) { + return _VSTD::__format_arg_store<_Context, _Args...>(__args...); } # ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS template _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI __format_arg_store -make_wformat_args(_Args&&... __args) { - return std::__format_arg_store(__args...); +make_wformat_args(_Args&... __args) { + return _VSTD::__format_arg_store(__args...); } # endif diff --git a/libcxx/include/__locale b/libcxx/include/__locale index ab36939f9c14c168f09e3023951d11b9bece8dfd..6eb892287160e15b7672608ef02263eb56c17ff1 100644 --- a/libcxx/include/__locale +++ b/libcxx/include/__locale @@ -389,7 +389,7 @@ public: static const mask punct = _ISpunct; static const mask xdigit = _ISxdigit; static const mask blank = _ISblank; -#if defined(__mips__) +#if defined(__mips__) || (BYTE_ORDER == BIG_ENDIAN) static const mask __regex_word = static_cast(_ISbit(15)); #else static const mask __regex_word = 0x80; diff --git a/libcxx/include/__mdspan/mdspan.h b/libcxx/include/__mdspan/mdspan.h index 58f3b9cf1b18a45aecab6d3e1f0c07252c5355f3..684828eb90ec7cd38dda303895d6c0992af8c202 100644 --- a/libcxx/include/__mdspan/mdspan.h +++ b/libcxx/include/__mdspan/mdspan.h @@ -244,9 +244,14 @@ public: _LIBCPP_HIDE_FROM_ABI constexpr const mapping_type& mapping() const noexcept { return __map_; }; _LIBCPP_HIDE_FROM_ABI constexpr const accessor_type& accessor() const noexcept { return __acc_; }; - _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_unique() { return mapping_type::is_always_unique(); }; - _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_exhaustive() { return mapping_type::is_always_exhaustive(); }; - _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_strided() { return mapping_type::is_always_strided(); }; + // per LWG-4021 "mdspan::is_always_meow() should be noexcept" + _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_unique() noexcept { return mapping_type::is_always_unique(); }; + _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_exhaustive() noexcept { + return mapping_type::is_always_exhaustive(); + }; + _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_strided() noexcept { + return mapping_type::is_always_strided(); + }; _LIBCPP_HIDE_FROM_ABI constexpr bool is_unique() const { return __map_.is_unique(); }; _LIBCPP_HIDE_FROM_ABI constexpr bool is_exhaustive() const { return __map_.is_exhaustive(); }; diff --git a/libcxx/include/__random/cauchy_distribution.h b/libcxx/include/__random/cauchy_distribution.h index b057074ab8d8f36f97dabd1f290b2161efd87294..44d7e0e27c14d3bf83fab99f1ed357d70c9200e7 100644 --- a/libcxx/include/__random/cauchy_distribution.h +++ b/libcxx/include/__random/cauchy_distribution.h @@ -28,6 +28,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS cauchy_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/chi_squared_distribution.h b/libcxx/include/__random/chi_squared_distribution.h index 5cae56b264178aa11da27400a1d7bef29d41b35a..ce17a283d4955e51709d7448d098ba89237d1e10 100644 --- a/libcxx/include/__random/chi_squared_distribution.h +++ b/libcxx/include/__random/chi_squared_distribution.h @@ -11,6 +11,7 @@ #include <__config> #include <__random/gamma_distribution.h> +#include <__random/is_valid.h> #include #include @@ -26,6 +27,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS chi_squared_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/exponential_distribution.h b/libcxx/include/__random/exponential_distribution.h index e5f0dad8cb88241cf9b2ab327747f675fea796a1..e9df4de90e747adf01f9643a77ceb7f28a8527f5 100644 --- a/libcxx/include/__random/exponential_distribution.h +++ b/libcxx/include/__random/exponential_distribution.h @@ -29,6 +29,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS exponential_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/extreme_value_distribution.h b/libcxx/include/__random/extreme_value_distribution.h index 4afc275981575ca96fd588a1c4b86fa9aaa0efbb..f8f18ebf7896c8f37a9fb8f62b74576f73dbec53 100644 --- a/libcxx/include/__random/extreme_value_distribution.h +++ b/libcxx/include/__random/extreme_value_distribution.h @@ -28,6 +28,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS extreme_value_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/fisher_f_distribution.h b/libcxx/include/__random/fisher_f_distribution.h index 1915cdb06a84338c0856d4bab3d48f716f07ca43..8da689a3b0f4f3bfe634bdc7f77d472308b60ca3 100644 --- a/libcxx/include/__random/fisher_f_distribution.h +++ b/libcxx/include/__random/fisher_f_distribution.h @@ -27,6 +27,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS fisher_f_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/gamma_distribution.h b/libcxx/include/__random/gamma_distribution.h index e2f04ad51727e9835250ff7e78aa779f0c6e0731..bdf8e141ece73c37b1f182fabfc3720e22c3722e 100644 --- a/libcxx/include/__random/gamma_distribution.h +++ b/libcxx/include/__random/gamma_distribution.h @@ -29,6 +29,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS gamma_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/is_valid.h b/libcxx/include/__random/is_valid.h index 113452bc5d35aa292207acd297ab0f0481a6952a..bde4733099d68843ef7b10f66e8c104d93f93be4 100644 --- a/libcxx/include/__random/is_valid.h +++ b/libcxx/include/__random/is_valid.h @@ -23,6 +23,20 @@ _LIBCPP_BEGIN_NAMESPACE_STD +// [rand.req.genl]/1.4: +// The effect of instantiating a template that has a template type parameter +// named RealType is undefined unless the corresponding template argument is +// cv-unqualified and is one of float, double, or long double. + +template +struct __libcpp_random_is_valid_realtype : false_type {}; +template <> +struct __libcpp_random_is_valid_realtype : true_type {}; +template <> +struct __libcpp_random_is_valid_realtype : true_type {}; +template <> +struct __libcpp_random_is_valid_realtype : true_type {}; + // [rand.req.genl]/1.5: // The effect of instantiating a template that has a template type parameter // named IntType is undefined unless the corresponding template argument is diff --git a/libcxx/include/__random/lognormal_distribution.h b/libcxx/include/__random/lognormal_distribution.h index c30f9adf103a1729584dcd979b8dfa8d8487ac55..47db56ee4529818298d9bcd3164ca8ca431e8a5c 100644 --- a/libcxx/include/__random/lognormal_distribution.h +++ b/libcxx/include/__random/lognormal_distribution.h @@ -10,6 +10,7 @@ #define _LIBCPP___RANDOM_LOGNORMAL_DISTRIBUTION_H #include <__config> +#include <__random/is_valid.h> #include <__random/normal_distribution.h> #include #include @@ -27,6 +28,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS lognormal_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/normal_distribution.h b/libcxx/include/__random/normal_distribution.h index 14649f1757ff61f9462ba0bac2f9e1d32817e1fa..9d68ea6118fd00f2671c3206be4b5636719ae430 100644 --- a/libcxx/include/__random/normal_distribution.h +++ b/libcxx/include/__random/normal_distribution.h @@ -28,6 +28,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS normal_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/piecewise_constant_distribution.h b/libcxx/include/__random/piecewise_constant_distribution.h index f5b90290209bb060328116420b5585499aa3e9cb..3a60e5533d773d8be8ab90757622ee9ed7695ca4 100644 --- a/libcxx/include/__random/piecewise_constant_distribution.h +++ b/libcxx/include/__random/piecewise_constant_distribution.h @@ -29,6 +29,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS piecewise_constant_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/piecewise_linear_distribution.h b/libcxx/include/__random/piecewise_linear_distribution.h index 824e089b0c3492c857e6b6836a58c851af136e3e..895f0f2dfe94988a7b7c8d5119a7faee0b93ae61 100644 --- a/libcxx/include/__random/piecewise_linear_distribution.h +++ b/libcxx/include/__random/piecewise_linear_distribution.h @@ -29,6 +29,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS piecewise_linear_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/student_t_distribution.h b/libcxx/include/__random/student_t_distribution.h index d52510dca4ed7093b4804d1f03ffe9c3b6c165ec..0879a411c0c5655c5e3e4ac44534d3101cc4ce3a 100644 --- a/libcxx/include/__random/student_t_distribution.h +++ b/libcxx/include/__random/student_t_distribution.h @@ -29,6 +29,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS student_t_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/uniform_real_distribution.h b/libcxx/include/__random/uniform_real_distribution.h index c5ee527353ab2b9258c421af4b16a144822a30aa..28b6e9181c5f330bab7e46763703c97a5fc64202 100644 --- a/libcxx/include/__random/uniform_real_distribution.h +++ b/libcxx/include/__random/uniform_real_distribution.h @@ -27,6 +27,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS uniform_real_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/weibull_distribution.h b/libcxx/include/__random/weibull_distribution.h index 099cfdd6171b14e45f377f4c2503f616e54d80ab..9967992810310928848a6aa5bf4788522cca2ccc 100644 --- a/libcxx/include/__random/weibull_distribution.h +++ b/libcxx/include/__random/weibull_distribution.h @@ -11,6 +11,7 @@ #include <__config> #include <__random/exponential_distribution.h> +#include <__random/is_valid.h> #include #include #include @@ -27,6 +28,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS weibull_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/any b/libcxx/include/any index 516fd6ddb23cc41c55c043828a27e7a062ecf1f7..7dcffc46a60d55eef8ec0f489ea1d5d4505ababe 100644 --- a/libcxx/include/any +++ b/libcxx/include/any @@ -194,9 +194,7 @@ namespace __any_imp if (__id && *__id == typeid(_Tp)) return true; #endif - if (!__id && __fallback_id == __any_imp::__get_fallback_typeid<_Tp>()) - return true; - return false; + return !__id && __fallback_id == __any_imp::__get_fallback_typeid<_Tp>(); } template diff --git a/libcxx/include/format b/libcxx/include/format index 7b8d5922cb4971ef409a76b00e80342871543d36..ab9b336d0cdabee3bffdcc8a2bf7f502d1490d4e 100644 --- a/libcxx/include/format +++ b/libcxx/include/format @@ -177,10 +177,10 @@ namespace std { template format-arg-store - make_format_args(Args&&... args); + make_format_args(Args&... args); template format-arg-store - make_wformat_args(Args&&... args); + make_wformat_args(Args&... args); // [format.error], class format_error class format_error; diff --git a/libcxx/include/mdspan b/libcxx/include/mdspan index d55cdc4a4df5c34b476eb170d6495dc59592b1dc..c13d9eef001ac9e42f3fe5a0ec84a985bd00988d 100644 --- a/libcxx/include/mdspan +++ b/libcxx/include/mdspan @@ -334,11 +334,12 @@ namespace std { constexpr const mapping_type& mapping() const noexcept { return map_; } constexpr const accessor_type& accessor() const noexcept { return acc_; } - static constexpr bool is_always_unique() + // per LWG-4021 "mdspan::is_always_meow() should be noexcept" + static constexpr bool is_always_unique() noexcept { return mapping_type::is_always_unique(); } - static constexpr bool is_always_exhaustive() + static constexpr bool is_always_exhaustive() noexcept { return mapping_type::is_always_exhaustive(); } - static constexpr bool is_always_strided() + static constexpr bool is_always_strided() noexcept { return mapping_type::is_always_strided(); } constexpr bool is_unique() const diff --git a/libcxx/modules/CMakeLists.txt b/libcxx/modules/CMakeLists.txt index 395226fb3728700cdad688931bcdb46851728aee..fae6448a7eec84bdf6b8054f493c66a033ce9de7 100644 --- a/libcxx/modules/CMakeLists.txt +++ b/libcxx/modules/CMakeLists.txt @@ -118,6 +118,30 @@ set(LIBCXX_MODULE_STD_SOURCES std/version.inc ) +set(LIBCXX_MODULE_STD_COMPAT_SOURCES + std.compat/cassert.inc + std.compat/cctype.inc + std.compat/cerrno.inc + std.compat/cfenv.inc + std.compat/cfloat.inc + std.compat/cinttypes.inc + std.compat/climits.inc + std.compat/clocale.inc + std.compat/cmath.inc + std.compat/csetjmp.inc + std.compat/csignal.inc + std.compat/cstdarg.inc + std.compat/cstddef.inc + std.compat/cstdint.inc + std.compat/cstdio.inc + std.compat/cstdlib.inc + std.compat/cstring.inc + std.compat/ctime.inc + std.compat/cuchar.inc + std.compat/cwchar.inc + std.compat/cwctype.inc +) + # TODO MODULES the CMakeLists.txt in the install directory is only temporary # When that is removed the configured file can use the substitution # LIBCXX_GENERATED_INCLUDE_TARGET_DIR avoiding this set. @@ -154,10 +178,25 @@ configure_file( @ONLY ) +set(LIBCXX_MODULE_STD_COMPAT_INCLUDE_SOURCES) +foreach(file ${LIBCXX_MODULE_STD_COMPAT_SOURCES}) + set( + LIBCXX_MODULE_STD_COMPAT_INCLUDE_SOURCES + "${LIBCXX_MODULE_STD_COMPAT_INCLUDE_SOURCES}#include \"${file}\"\n" + ) +endforeach() + +configure_file( + "std.compat.cppm.in" + "${LIBCXX_GENERATED_MODULE_DIR}/std.compat.cppm" + @ONLY +) + set(_all_modules) list(APPEND _all_modules "${LIBCXX_GENERATED_MODULE_DIR}/CMakeLists.txt") list(APPEND _all_modules "${LIBCXX_GENERATED_MODULE_DIR}/std.cppm") -foreach(file ${LIBCXX_MODULE_STD_SOURCES}) +list(APPEND _all_modules "${LIBCXX_GENERATED_MODULE_DIR}/std.compat.cppm") +foreach(file ${LIBCXX_MODULE_STD_SOURCES} ${LIBCXX_MODULE_STD_COMPAT_SOURCES}) set(src "${CMAKE_CURRENT_SOURCE_DIR}/${file}") set(dst "${LIBCXX_GENERATED_MODULE_DIR}/${file}") add_custom_command(OUTPUT ${dst} diff --git a/libcxx/modules/CMakeLists.txt.in b/libcxx/modules/CMakeLists.txt.in index dca3b25155a5af469d8f5e187218bbd7ae3dba92..b02b68915b8f4d0fd669c43daadb4d758f00733e 100644 --- a/libcxx/modules/CMakeLists.txt.in +++ b/libcxx/modules/CMakeLists.txt.in @@ -29,6 +29,8 @@ macro(compile_define_if condition def) endif() endmacro() +### STD + add_library(std) target_sources(std PUBLIC FILE_SET cxx_modules TYPE CXX_MODULES FILES @@ -52,3 +54,29 @@ set_target_properties(std PROPERTIES OUTPUT_NAME "c++std" ) + +### STD.COMPAT + +add_library(std.compat) +target_sources(std.compat + PUBLIC FILE_SET cxx_modules TYPE CXX_MODULES FILES + std.compat.cppm +) + +target_include_directories(std.compat SYSTEM PRIVATE @LIBCXX_CONFIGURED_INCLUDE_DIRS@) + +if (NOT @LIBCXX_ENABLE_EXCEPTIONS@) + target_compile_options(std.compat PUBLIC -fno-exceptions) +endif() + +target_compile_options(std.compat + PUBLIC + -nostdinc++ + -Wno-reserved-module-identifier + -Wno-reserved-user-defined-literal + @LIBCXX_COMPILE_FLAGS@ +) +set_target_properties(std.compat + PROPERTIES + OUTPUT_NAME "c++std.compat" +) diff --git a/libcxx/modules/std.compat.cppm.in b/libcxx/modules/std.compat.cppm.in new file mode 100644 index 0000000000000000000000000000000000000000..f199e194e60b163df377ce269a55a06ef4747a6a --- /dev/null +++ b/libcxx/modules/std.compat.cppm.in @@ -0,0 +1,208 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// WARNING, this entire header is generated by +// utils/generate_libcxx_cppm_in.py +// DO NOT MODIFY! + +module; + +#include <__config> + +// The headers of Table 24: C++ library headers [tab:headers.cpp] +// and the headers of Table 25: C++ headers for C library facilities [tab:headers.cpp.c] +#include +#include +#include +#if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER) +# include +#endif +#if !defined(_LIBCPP_HAS_NO_THREADS) +# include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS) +# include +#endif +#if !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS) +# include +#endif +#include +#include +#include +#include +#include +#include +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_THREADS) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_THREADS) +# include +#endif +#include +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#include +#include +#include +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_THREADS) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_THREADS) +# include +#endif +#include +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#include +#if !defined(_LIBCPP_HAS_NO_THREADS) +# include +#endif +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_THREADS) +# include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// *** Headers not yet available *** +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() + +export module std.compat; + +@LIBCXX_MODULE_STD_INCLUDE_SOURCES@ +@LIBCXX_MODULE_STD_COMPAT_INCLUDE_SOURCES@ \ No newline at end of file diff --git a/libcxx/modules/std.compat/cassert.inc b/libcxx/modules/std.compat/cassert.inc new file mode 100644 index 0000000000000000000000000000000000000000..ac0533d14e9a9a449f345f6639eed536d6b48cc5 --- /dev/null +++ b/libcxx/modules/std.compat/cassert.inc @@ -0,0 +1,12 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + // This module exports nothing. +} // export diff --git a/libcxx/modules/std.compat/cctype.inc b/libcxx/modules/std.compat/cctype.inc new file mode 100644 index 0000000000000000000000000000000000000000..56fb45a374a510a6fc892b833229f2b1184e2d80 --- /dev/null +++ b/libcxx/modules/std.compat/cctype.inc @@ -0,0 +1,25 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::isalnum; + using ::isalpha; + using ::isblank; + using ::iscntrl; + using ::isdigit; + using ::isgraph; + using ::islower; + using ::isprint; + using ::ispunct; + using ::isspace; + using ::isupper; + using ::isxdigit; + using ::tolower; + using ::toupper; +} // export diff --git a/libcxx/modules/std.compat/cerrno.inc b/libcxx/modules/std.compat/cerrno.inc new file mode 100644 index 0000000000000000000000000000000000000000..ac0533d14e9a9a449f345f6639eed536d6b48cc5 --- /dev/null +++ b/libcxx/modules/std.compat/cerrno.inc @@ -0,0 +1,12 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + // This module exports nothing. +} // export diff --git a/libcxx/modules/std.compat/cfenv.inc b/libcxx/modules/std.compat/cfenv.inc new file mode 100644 index 0000000000000000000000000000000000000000..50128463d6a91433504184c8a99b87347a44cf02 --- /dev/null +++ b/libcxx/modules/std.compat/cfenv.inc @@ -0,0 +1,29 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + // types + using ::fenv_t; + using ::fexcept_t; + + // functions + using ::feclearexcept; + using ::fegetexceptflag; + using ::feraiseexcept; + using ::fesetexceptflag; + using ::fetestexcept; + + using ::fegetround; + using ::fesetround; + + using ::fegetenv; + using ::feholdexcept; + using ::fesetenv; + using ::feupdateenv; +} // export diff --git a/libcxx/modules/std.compat/cfloat.inc b/libcxx/modules/std.compat/cfloat.inc new file mode 100644 index 0000000000000000000000000000000000000000..ac0533d14e9a9a449f345f6639eed536d6b48cc5 --- /dev/null +++ b/libcxx/modules/std.compat/cfloat.inc @@ -0,0 +1,12 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + // This module exports nothing. +} // export diff --git a/libcxx/modules/std.compat/cinttypes.inc b/libcxx/modules/std.compat/cinttypes.inc new file mode 100644 index 0000000000000000000000000000000000000000..a64c088d0d6f881caf59f520a0c545cdeda0b53c --- /dev/null +++ b/libcxx/modules/std.compat/cinttypes.inc @@ -0,0 +1,25 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::imaxdiv_t; + + using ::imaxabs; + using ::imaxdiv; + using ::strtoimax; + using ::strtoumax; + using ::wcstoimax; + using ::wcstoumax; + + // abs is conditionally here, but always present in cmath.cppm. To avoid + // conflicing declarations omit the using here. + + // div is conditionally here, but always present in cstdlib.cppm. To avoid + // conflicing declarations omit the using here. +} // export diff --git a/libcxx/modules/std.compat/climits.inc b/libcxx/modules/std.compat/climits.inc new file mode 100644 index 0000000000000000000000000000000000000000..ac0533d14e9a9a449f345f6639eed536d6b48cc5 --- /dev/null +++ b/libcxx/modules/std.compat/climits.inc @@ -0,0 +1,12 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + // This module exports nothing. +} // export diff --git a/libcxx/modules/std.compat/clocale.inc b/libcxx/modules/std.compat/clocale.inc new file mode 100644 index 0000000000000000000000000000000000000000..d9785a73794300c06bb69f0b93ac498d25ab8270 --- /dev/null +++ b/libcxx/modules/std.compat/clocale.inc @@ -0,0 +1,17 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { +#ifndef _LIBCPP_HAS_NO_LOCALIZATION + using ::lconv; + + using ::localeconv; + using ::setlocale; +#endif // _LIBCPP_HAS_NO_LOCALIZATION +} // export diff --git a/libcxx/modules/std.compat/cmath.inc b/libcxx/modules/std.compat/cmath.inc new file mode 100644 index 0000000000000000000000000000000000000000..de5379275c5fae6fa75a2ab49f6e69eb84818a25 --- /dev/null +++ b/libcxx/modules/std.compat/cmath.inc @@ -0,0 +1,268 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::double_t; + using ::float_t; + + using ::acos; + using ::acosf; + using ::acosl; + + using ::asin; + using ::asinf; + using ::asinl; + + using ::atan; + using ::atanf; + using ::atanl; + + using ::atan2; + using ::atan2f; + using ::atan2l; + + using ::cos; + using ::cosf; + using ::cosl; + + using ::sin; + using ::sinf; + using ::sinl; + + using ::tan; + using ::tanf; + using ::tanl; + + using ::acosh; + using ::acoshf; + using ::acoshl; + + using ::asinh; + using ::asinhf; + using ::asinhl; + + using ::atanh; + using ::atanhf; + using ::atanhl; + + using ::cosh; + using ::coshf; + using ::coshl; + + using ::sinh; + using ::sinhf; + using ::sinhl; + + using ::tanh; + using ::tanhf; + using ::tanhl; + + using ::exp; + using ::expf; + using ::expl; + + using ::exp2; + using ::exp2f; + using ::exp2l; + + using ::expm1; + using ::expm1f; + using ::expm1l; + + using ::frexp; + using ::frexpf; + using ::frexpl; + + using ::ilogb; + using ::ilogbf; + using ::ilogbl; + + using ::ldexp; + using ::ldexpf; + using ::ldexpl; + + using ::log; + using ::logf; + using ::logl; + + using ::log10; + using ::log10f; + using ::log10l; + + using ::log1p; + using ::log1pf; + using ::log1pl; + + using ::log2; + using ::log2f; + using ::log2l; + + using ::logb; + using ::logbf; + using ::logbl; + + using ::modf; + using ::modff; + using ::modfl; + + using ::scalbn; + using ::scalbnf; + using ::scalbnl; + + using ::scalbln; + using ::scalblnf; + using ::scalblnl; + + using ::cbrt; + using ::cbrtf; + using ::cbrtl; + + // [c.math.abs], absolute values + using ::abs; + + using ::fabs; + using ::fabsf; + using ::fabsl; + + using ::hypot; + using ::hypotf; + using ::hypotl; + + // [c.math.hypot3], three-dimensional hypotenuse + + using ::pow; + using ::powf; + using ::powl; + + using ::sqrt; + using ::sqrtf; + using ::sqrtl; + + using ::erf; + using ::erff; + using ::erfl; + + using ::erfc; + using ::erfcf; + using ::erfcl; + + using ::lgamma; + using ::lgammaf; + using ::lgammal; + + using ::tgamma; + using ::tgammaf; + using ::tgammal; + + using ::ceil; + using ::ceilf; + using ::ceill; + + using ::floor; + using ::floorf; + using ::floorl; + + using ::nearbyint; + using ::nearbyintf; + using ::nearbyintl; + + using ::rint; + using ::rintf; + using ::rintl; + + using ::lrint; + using ::lrintf; + using ::lrintl; + + using ::llrint; + using ::llrintf; + using ::llrintl; + + using ::round; + using ::roundf; + using ::roundl; + + using ::lround; + using ::lroundf; + using ::lroundl; + + using ::llround; + using ::llroundf; + using ::llroundl; + + using ::trunc; + using ::truncf; + using ::truncl; + + using ::fmod; + using ::fmodf; + using ::fmodl; + + using ::remainder; + using ::remainderf; + using ::remainderl; + + using ::remquo; + using ::remquof; + using ::remquol; + + using ::copysign; + using ::copysignf; + using ::copysignl; + + using ::nan; + using ::nanf; + using ::nanl; + + using ::nextafter; + using ::nextafterf; + using ::nextafterl; + + using ::nexttoward; + using ::nexttowardf; + using ::nexttowardl; + + using ::fdim; + using ::fdimf; + using ::fdiml; + + using ::fmax; + using ::fmaxf; + using ::fmaxl; + + using ::fmin; + using ::fminf; + using ::fminl; + + using ::fma; + using ::fmaf; + using ::fmal; + + // [c.math.lerp], linear interpolation + // [support.c.headers.other]/1 + // ... placed within the global namespace scope, except for the functions + // described in [sf.cmath], the std::lerp function overloads ([c.math.lerp]) + // ... + + // [c.math.fpclass], classification / comparison functions + using ::fpclassify; + using ::isfinite; + using ::isgreater; + using ::isgreaterequal; + using ::isinf; + using ::isless; + using ::islessequal; + using ::islessgreater; + using ::isnan; + using ::isnormal; + using ::isunordered; + using ::signbit; + + // [sf.cmath], mathematical special functions +} // export diff --git a/libcxx/modules/std.compat/csetjmp.inc b/libcxx/modules/std.compat/csetjmp.inc new file mode 100644 index 0000000000000000000000000000000000000000..1fc42ea3ee0378bb843843b98e62e161e1f2ddb6 --- /dev/null +++ b/libcxx/modules/std.compat/csetjmp.inc @@ -0,0 +1,13 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::jmp_buf; + using ::longjmp; +} // export diff --git a/libcxx/modules/std.compat/csignal.inc b/libcxx/modules/std.compat/csignal.inc new file mode 100644 index 0000000000000000000000000000000000000000..33af6a9f2b7343e1cba5a75f26df2fad9fcae9e4 --- /dev/null +++ b/libcxx/modules/std.compat/csignal.inc @@ -0,0 +1,17 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::sig_atomic_t; + + // [support.signal], signal handlers + using ::signal; + + using ::raise; +} // export diff --git a/libcxx/modules/std.compat/cstdarg.inc b/libcxx/modules/std.compat/cstdarg.inc new file mode 100644 index 0000000000000000000000000000000000000000..3efb34617a8bff2f2fee4eb6e844b6b2997803ae --- /dev/null +++ b/libcxx/modules/std.compat/cstdarg.inc @@ -0,0 +1,10 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { using ::va_list; } // export diff --git a/libcxx/modules/std.compat/cstddef.inc b/libcxx/modules/std.compat/cstddef.inc new file mode 100644 index 0000000000000000000000000000000000000000..94ad036fd8f4a5aef499f18604ea2fbebbd31a22 --- /dev/null +++ b/libcxx/modules/std.compat/cstddef.inc @@ -0,0 +1,22 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::max_align_t; + using ::nullptr_t; + using ::ptrdiff_t; + using ::size_t; + + // [support.c.headers]/1 + // ... placed within the global namespace scope, except for ... the + // declaration of std::byte ([cstddef.syn]), and the functions and + // function templates described in [support.types.byteops]. ... + + // [support.types.byteops], byte type operations +} // export diff --git a/libcxx/modules/std.compat/cstdint.inc b/libcxx/modules/std.compat/cstdint.inc new file mode 100644 index 0000000000000000000000000000000000000000..1a74efc70ceaa1a353a1a9a8153c975d3379bbb2 --- /dev/null +++ b/libcxx/modules/std.compat/cstdint.inc @@ -0,0 +1,50 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + // signed + using ::int8_t _LIBCPP_USING_IF_EXISTS; + using ::int16_t _LIBCPP_USING_IF_EXISTS; + using ::int32_t _LIBCPP_USING_IF_EXISTS; + using ::int64_t _LIBCPP_USING_IF_EXISTS; + + using ::int_fast16_t; + using ::int_fast32_t; + using ::int_fast64_t; + using ::int_fast8_t; + + using ::int_least16_t; + using ::int_least32_t; + using ::int_least64_t; + using ::int_least8_t; + + using ::intmax_t; + + using ::intptr_t _LIBCPP_USING_IF_EXISTS; + + // unsigned + using ::uint8_t _LIBCPP_USING_IF_EXISTS; + using ::uint16_t _LIBCPP_USING_IF_EXISTS; + using ::uint32_t _LIBCPP_USING_IF_EXISTS; + using ::uint64_t _LIBCPP_USING_IF_EXISTS; + + using ::uint_fast16_t; + using ::uint_fast32_t; + using ::uint_fast64_t; + using ::uint_fast8_t; + + using ::uint_least16_t; + using ::uint_least32_t; + using ::uint_least64_t; + using ::uint_least8_t; + + using ::uintmax_t; + + using ::uintptr_t _LIBCPP_USING_IF_EXISTS; +} // export diff --git a/libcxx/modules/std.compat/cstdio.inc b/libcxx/modules/std.compat/cstdio.inc new file mode 100644 index 0000000000000000000000000000000000000000..1ec3015c9e2a2e8b374d6541ecaf79fdcef666e5 --- /dev/null +++ b/libcxx/modules/std.compat/cstdio.inc @@ -0,0 +1,61 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::FILE; + using ::fpos_t; + using ::size_t; + + using ::clearerr; + using ::fclose; + using ::feof; + using ::ferror; + using ::fflush; + using ::fgetc; + using ::fgetpos; + using ::fgets; + using ::fopen; + using ::fprintf; + using ::fputc; + using ::fputs; + using ::fread; + using ::freopen; + using ::fscanf; + using ::fseek; + using ::fsetpos; + using ::ftell; + using ::fwrite; + using ::getc; + using ::getchar; + using ::perror; + using ::printf; + using ::putc; + using ::putchar; + using ::puts; + using ::remove; + using ::rename; + using ::rewind; + using ::scanf; + using ::setbuf; + using ::setvbuf; + using ::snprintf; + using ::sprintf; + using ::sscanf; + using ::tmpfile; + using ::tmpnam; + using ::ungetc; + using ::vfprintf; + using ::vfscanf; + using ::vprintf; + using ::vscanf; + using ::vsnprintf; + using ::vsprintf; + using ::vsscanf; + +} // export diff --git a/libcxx/modules/std.compat/cstdlib.inc b/libcxx/modules/std.compat/cstdlib.inc new file mode 100644 index 0000000000000000000000000000000000000000..9333d848707103da2e2d74525f6fbe1d32938ea4 --- /dev/null +++ b/libcxx/modules/std.compat/cstdlib.inc @@ -0,0 +1,72 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::div_t; + using ::ldiv_t; + using ::lldiv_t; + using ::size_t; + + // [support.start.term], start and termination + using ::_Exit; + using ::abort; + using ::at_quick_exit; + using ::atexit; + using ::exit; + using ::quick_exit; + + using ::getenv; + using ::system; + + // [c.malloc], C library memory allocation + using ::aligned_alloc; + using ::calloc; + using ::free; + using ::malloc; + using ::realloc; + + using ::atof; + using ::atoi; + using ::atol; + using ::atoll; + using ::strtod; + using ::strtof; + using ::strtol; + using ::strtold; + using ::strtoll; + using ::strtoul; + using ::strtoull; + + // [c.mb.wcs], multibyte / wide string and character conversion functions + using ::mblen; +#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS + using ::mbstowcs; + using ::mbtowc; + using ::wcstombs; + using ::wctomb; +#endif + // [alg.c.library], C standard library algorithms + using ::bsearch; + using ::qsort; + + // [c.math.rand], low-quality random number generation + using ::rand; + using ::srand; + + // [c.math.abs], absolute values + using ::abs; + + using ::labs; + using ::llabs; + + using ::div; + using ::ldiv; + using ::lldiv; + +} // export diff --git a/libcxx/modules/std.compat/cstring.inc b/libcxx/modules/std.compat/cstring.inc new file mode 100644 index 0000000000000000000000000000000000000000..090350ae814786abdc8da342e0f075a97f7125ce --- /dev/null +++ b/libcxx/modules/std.compat/cstring.inc @@ -0,0 +1,36 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::size_t; + + using ::memchr; + using ::memcmp; + using ::memcpy; + using ::memmove; + using ::memset; + using ::strcat; + using ::strchr; + using ::strcmp; + using ::strcoll; + using ::strcpy; + using ::strcspn; + using ::strerror; + using ::strlen; + using ::strncat; + using ::strncmp; + using ::strncpy; + using ::strpbrk; + using ::strrchr; + using ::strspn; + using ::strstr; + using ::strtok; + using ::strxfrm; + +} // export diff --git a/libcxx/modules/std.compat/ctime.inc b/libcxx/modules/std.compat/ctime.inc new file mode 100644 index 0000000000000000000000000000000000000000..92e3403a5e58e274977a6cda55886b3a3ee9590e --- /dev/null +++ b/libcxx/modules/std.compat/ctime.inc @@ -0,0 +1,28 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::clock_t; + using ::size_t; + using ::time_t; + + using ::timespec; + using ::tm; + + using ::asctime; + using ::clock; + using ::ctime; + using ::difftime; + using ::gmtime; + using ::localtime; + using ::mktime; + using ::strftime; + using ::time; + using ::timespec_get; +} // export diff --git a/libcxx/modules/std.compat/cuchar.inc b/libcxx/modules/std.compat/cuchar.inc new file mode 100644 index 0000000000000000000000000000000000000000..d1a511cadef18461122330f771b335de9d536e0a --- /dev/null +++ b/libcxx/modules/std.compat/cuchar.inc @@ -0,0 +1,28 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + // Note the Standard does not mark these symbols optional, but libc++'s header + // does. So this seems strictly not to be conforming. + + // mbstate_t is conditionally here, but always present in cwchar.cppm. To avoid + // conflicing declarations omit the using here. + + // size_t is conditionally here, but always present in cstddef.cppm. To avoid + // conflicing declarations omit the using here. + +#if !defined(_LIBCPP_HAS_NO_C8RTOMB_MBRTOC8) + using ::mbrtoc8 _LIBCPP_USING_IF_EXISTS; + using ::c8rtomb _LIBCPP_USING_IF_EXISTS; +#endif + using ::mbrtoc16 _LIBCPP_USING_IF_EXISTS; + using ::c16rtomb _LIBCPP_USING_IF_EXISTS; + using ::mbrtoc32 _LIBCPP_USING_IF_EXISTS; + using ::c32rtomb _LIBCPP_USING_IF_EXISTS; +} // export diff --git a/libcxx/modules/std.compat/cwchar.inc b/libcxx/modules/std.compat/cwchar.inc new file mode 100644 index 0000000000000000000000000000000000000000..8905aecbdfecc2a55b811c1a0a3034691946f406 --- /dev/null +++ b/libcxx/modules/std.compat/cwchar.inc @@ -0,0 +1,80 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { +#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS + using ::mbstate_t; + using ::size_t; + using ::wint_t; + + using ::tm; + + using ::btowc; + using ::fgetwc; + using ::fgetws; + using ::fputwc; + using ::fputws; + using ::fwide; + using ::fwprintf; + using ::fwscanf; + using ::getwc; + using ::getwchar; + using ::putwc; + using ::putwchar; + using ::swprintf; + using ::swscanf; + using ::ungetwc; + using ::vfwprintf; + using ::vfwscanf; + using ::vswprintf; + using ::vswscanf; + using ::vwprintf; + using ::vwscanf; + using ::wcscat; + using ::wcschr; + using ::wcscmp; + using ::wcscoll; + using ::wcscpy; + using ::wcscspn; + using ::wcsftime; + using ::wcslen; + using ::wcsncat; + using ::wcsncmp; + using ::wcsncpy; + using ::wcspbrk; + using ::wcsrchr; + using ::wcsspn; + using ::wcsstr; + using ::wcstod; + using ::wcstof; + using ::wcstok; + using ::wcstol; + using ::wcstold; + using ::wcstoll; + using ::wcstoul; + using ::wcstoull; + using ::wcsxfrm; + using ::wctob; + using ::wmemchr; + using ::wmemcmp; + using ::wmemcpy; + using ::wmemmove; + using ::wmemset; + using ::wprintf; + using ::wscanf; + + // [c.mb.wcs], multibyte / wide string and character conversion functions + using ::mbrlen; + using ::mbrtowc; + using ::mbsinit; + using ::mbsrtowcs; + using ::wcrtomb; + using ::wcsrtombs; +#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS +} // export diff --git a/libcxx/modules/std.compat/cwctype.inc b/libcxx/modules/std.compat/cwctype.inc new file mode 100644 index 0000000000000000000000000000000000000000..13aa2b7f3fb74611ea92df65a5db73a98396a422 --- /dev/null +++ b/libcxx/modules/std.compat/cwctype.inc @@ -0,0 +1,35 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { +#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS + using ::wctrans_t; + using ::wctype_t; + using ::wint_t; + + using ::iswalnum; + using ::iswalpha; + using ::iswblank; + using ::iswcntrl; + using ::iswctype; + using ::iswdigit; + using ::iswgraph; + using ::iswlower; + using ::iswprint; + using ::iswpunct; + using ::iswspace; + using ::iswupper; + using ::iswxdigit; + using ::towctrans; + using ::towlower; + using ::towupper; + using ::wctrans; + using ::wctype; +#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS +} // export diff --git a/libcxx/modules/std.cppm.in b/libcxx/modules/std.cppm.in index ecb06012603253b146b2b6728082d85757944c33..b46c52e781f82f9ccd88ecdbc9f13a4b26593148 100644 --- a/libcxx/modules/std.cppm.in +++ b/libcxx/modules/std.cppm.in @@ -8,7 +8,7 @@ //===----------------------------------------------------------------------===// // WARNING, this entire header is generated by -// utils/generate_std_cppm_in.py +// utils/generate_libcxx_cppm_in.py // DO NOT MODIFY! module; @@ -169,38 +169,38 @@ module; // *** Headers not yet available *** #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() export module std; diff --git a/libcxx/test/libcxx/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp b/libcxx/test/libcxx/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp index 42a7c08fa75ba260d71eadd8508b5b04d1aec93a..a59839ff806eae5be535b4fa9dcf1c4c3c4f7763 100644 --- a/libcxx/test/libcxx/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp +++ b/libcxx/test/libcxx/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp @@ -45,7 +45,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/libcxx/module_std.gen.py b/libcxx/test/libcxx/module_std.gen.py index d8198cc7058379e1a503c0ae982535552e917ea5..8e03d6e5b5b5235cb68bb3d0548dae04c82fdd2e 100644 --- a/libcxx/test/libcxx/module_std.gen.py +++ b/libcxx/test/libcxx/module_std.gen.py @@ -21,241 +21,17 @@ import sys sys.path.append(sys.argv[1]) -from libcxx.header_information import module_headers -from libcxx.header_information import header_restrictions - -BLOCKLIT = ( - "" # block Lit from interpreting a RUN/XFAIL/etc inside the generation script +from libcxx.test.modules import module_test_generator + +generator = module_test_generator( + "%t", + "%{module}", + "%{clang-tidy}", + "%{test-tools}/clang_tidy_checks/libcxx-tidy.plugin", + "%{cxx}", + "%{flags} %{compile_flags}", ) -# Ignore several declarations found in the includes. -# -# Part of these items are bugs other are not yet implemented features. -SkipDeclarations = dict() - -# See comment in the header. -SkipDeclarations["cuchar"] = ["std::mbstate_t", "std::size_t"] - -# Not in the synopsis. -SkipDeclarations["cwchar"] = ["std::FILE"] - -# The operators are added for private types like __iom_t10. -SkipDeclarations["iomanip"] = ["std::operator<<", "std::operator>>"] - -SkipDeclarations["iosfwd"] = ["std::ios_base", "std::vector"] - -# This header also provides declarations in the namespace that might be -# an error. -SkipDeclarations["filesystem"] = [ - "std::filesystem::operator==", - "std::filesystem::operator!=", -] - -# This is a specialization for a private type -SkipDeclarations["iterator"] = ["std::pointer_traits"] - -# TODO MODULES -# This definition is declared in string and defined in istream -# This declaration should be part of string -SkipDeclarations["istream"] = ["std::getline"] - -# P1614 (at many places) and LWG3519 too. -SkipDeclarations["random"] = [ - "std::operator!=", - # LWG3519 makes these hidden friends. - # Note the older versions had the requirement of these operations but not in - # the synopsis. - "std::operator<<", - "std::operator>>", - "std::operator==", -] - -# Declared in the forward header since std::string uses std::allocator -SkipDeclarations["string"] = ["std::allocator"] -# TODO MODULES remove zombie names -# https://libcxx.llvm.org/Status/Cxx20.html#note-p0619 -SkipDeclarations["memory"] = [ - "std::return_temporary_buffer", - "std::get_temporary_buffer", -] - -# TODO MODULES this should be part of ios instead -SkipDeclarations["streambuf"] = ["std::basic_ios"] - -# include/__type_traits/is_swappable.h -SkipDeclarations["type_traits"] = [ - "std::swap", - # TODO MODULES gotten through __functional/unwrap_ref.h - "std::reference_wrapper", -] - -# Add declarations in headers. -# -# Some headers have their defines in a different header, which may have -# additional declarations. -ExtraDeclarations = dict() -# This declaration is in the ostream header. -ExtraDeclarations["system_error"] = ["std::operator<<"] - -# Adds an extra header file to scan -# -# -ExtraHeader = dict() -# locale has a file and not a subdirectory -ExtraHeader["locale"] = "v1/__locale$" -ExtraHeader["thread"] = "v1/__threading_support$" -ExtraHeader["ranges"] = "v1/__fwd/subrange.h$" - -# The extra header is needed since two headers are required to provide the -# same definition. -ExtraHeader["functional"] = "v1/__compare/compare_three_way.h$" - -# newline needs to be escaped for the module partition output. -nl = '\\\\n' - -# Create empty file with all parts. -print( - f"""\ -//--- module_std.sh.cpp -// UNSUPPORTED{BLOCKLIT}: c++03, c++11, c++14, c++17 -// UNSUPPORTED{BLOCKLIT}: libcpp-has-no-std-modules -// UNSUPPORTED{BLOCKLIT}: clang-modules-build - -// REQUIRES{BLOCKLIT}: has-clang-tidy - -// The GCC compiler flags are not always compatible with clang-tidy. -// UNSUPPORTED{BLOCKLIT}: gcc - -// RUN{BLOCKLIT}: echo -n > %t.all_partitions -""" -) - -# Validate all module parts. -for header in module_headers: - # Some headers cannot be included when a libc++ feature is disabled. - # In that case include the header conditionally. The header __config - # ensures the libc++ feature macros are available. - if header in header_restrictions: - include = ( - f"#include <__config>{nl}" - + f"#if {header_restrictions[header]}{nl}" - + f"# include <{header}>{nl}" - + f"#endif{nl}" - ) - elif header == "chrono": - # When localization is disabled the header string is not included. - # When string is included chrono's operator""s is a named declaration - # using std::chrono_literals::operator""s; - # else it is a named declaration - # using std::operator""s; - # TODO MODULES investigate why - include = f"#include {nl}#include {nl}" - else: - include = f"#include <{header}>{nl}" - - # Generate a module partition for the header module includes. This - # makes it possible to verify that all headers export all their - # named declarations. - print( - f"// RUN{BLOCKLIT}: echo -e \"" - f"module;{nl}" - f"{include}" - f"{nl}" - f"// Use __libcpp_module_
to ensure that modules {nl}" - f"// are not named as keywords or reserved names.{nl}" - f"export module std:__libcpp_module_{header};{nl}" - f'#include \\"%{{module}}/std/{header}.inc\\"{nl}' - f"\" > %t.{header}.cppm") - - # Dump the information as found in the module's cppm file. - print( - f"// RUN{BLOCKLIT}: %{{clang-tidy}} %t.{header}.cppm " - " --checks='-*,libcpp-header-exportable-declarations' " - " -config='{CheckOptions: [ " - " {" - " key: libcpp-header-exportable-declarations.Filename, " - f" value: {header}.inc" - " }, {" - " key: libcpp-header-exportable-declarations.FileType, " - " value: ModulePartition" - " }, " - " ]}' " - " --load=%{test-tools}/clang_tidy_checks/libcxx-tidy.plugin " - " -- %{flags} %{compile_flags} " - f"| sort > %t.{header}.module" - ) - print(f"// RUN{BLOCKLIT}: cat %t.{header}.module >> %t.all_partitions") - - # Dump the information as found in the module by using the header file(s). - skip_declarations = " ".join(SkipDeclarations.get(header, [])) - if skip_declarations: - skip_declarations = ( - "{" - " key: libcpp-header-exportable-declarations.SkipDeclarations, " - f' value: "{skip_declarations}" ' - "}, " - ) - - extra_declarations = " ".join(ExtraDeclarations.get(header, [])) - if extra_declarations: - extra_declarations = ( - " {" - " key: libcpp-header-exportable-declarations.ExtraDeclarations, " - f' value: "{extra_declarations}" ' - "}, " - ) - - extra_header = ExtraHeader.get(header, "") - if extra_header: - extra_header = ( - "{" - " key: libcpp-header-exportable-declarations.ExtraHeader, " - f' value: "{extra_header}" ' - "}, " - ) - - # Clang-tidy needs a file input - print(f'// RUN{BLOCKLIT}: echo -e "' f"{include}" f'" > %t.{header}.cpp') - print( - f"// RUN{BLOCKLIT}: %{{clang-tidy}} %t.{header}.cpp " - " --checks='-*,libcpp-header-exportable-declarations' " - " -config='{CheckOptions: [ " - f" {{key: libcpp-header-exportable-declarations.Filename, value: {header}}}, " - " {key: libcpp-header-exportable-declarations.FileType, value: Header}, " - f" {skip_declarations} {extra_declarations} {extra_header}, " - " ]}' " - " --load=%{test-tools}/clang_tidy_checks/libcxx-tidy.plugin " - " -- %{flags} %{compile_flags} " - f" | sort > %t.{header}.include" - ) - - # Compare the cppm and header file(s) return the same results. - print(f"// RUN{BLOCKLIT}: diff -u %t.{header}.module %t.{header}.include") - - -# Merge the data of the parts -print(f"// RUN{BLOCKLIT}: sort -u -o %t.all_partitions %t.all_partitions") - -# Dump the information as found in std.cppm. -print( - f"// RUN{BLOCKLIT}: %{{clang-tidy}} %{{module}}/std.cppm " - " --checks='-*,libcpp-header-exportable-declarations' " - " -config='{CheckOptions: [ " - " {key: libcpp-header-exportable-declarations.Header, value: std.cppm}, " - " {key: libcpp-header-exportable-declarations.FileType, value: Module}, " - " ]}' " - f" --load=%{{test-tools}}/clang_tidy_checks/libcxx-tidy.plugin " - " -- %{flags} %{compile_flags} " - " | sort > %t.module" -) - - -# Compare the sum of the parts with the main module. -print(f"// RUN{BLOCKLIT}: diff -u %t.all_partitions %t.module") -# Basic smoke test. Import a module and try to compile when using all -# exported names. This validates the clang-tidy script does not accidentally -# add named declarations to the list that are not available. -print(f"// RUN{BLOCKLIT}: echo 'import std;' > %t.compile.pass.cpp") -print(f"// RUN{BLOCKLIT}: cat %t.all_partitions >> %t.compile.pass.cpp") -print(f"// RUN{BLOCKLIT}: %{{cxx}} %{{flags}} %{{compile_flags}} -fsyntax-only %t.compile.pass.cpp") +print("//--- module_std.sh.cpp") +generator.write_test("std") diff --git a/libcxx/test/libcxx/module_std_compat.gen.py b/libcxx/test/libcxx/module_std_compat.gen.py new file mode 100644 index 0000000000000000000000000000000000000000..c4792db3d71e624643560aa65fbee89fb750bf79 --- /dev/null +++ b/libcxx/test/libcxx/module_std_compat.gen.py @@ -0,0 +1,62 @@ +# ===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------===## + +# Test that all named declarations with external linkage match the +# exported declarations in their associated module partition. +# Then it tests the sum of the exported declarations in the module +# partitions matches the export of the std.compat module. + +# Note the test of the std.compat module requires all partitions to be tested +# first. Since lit tests have no dependencies, this means the test needs +# to be one monolitic test. Since the test doesn't take very long it's +# not a huge issue. + +# RUN: %{python} %s %{libcxx}/utils + +import sys + +sys.path.append(sys.argv[1]) +from libcxx.test.modules import module_test_generator + +generator = module_test_generator( + "%t", + "%{module}", + "%{clang-tidy}", + "%{test-tools}/clang_tidy_checks/libcxx-tidy.plugin", + "%{cxx}", + "%{flags} %{compile_flags}", +) + + +print("//--- module_std_compat.sh.cpp") +generator.write_test( + "std.compat", + [ + "cassert", + "cctype", + "cerrno", + "cfenv", + "cfloat", + "cinttypes", + "climits", + "clocale", + "cmath", + "csetjmp", + "csignal", + "cstdarg", + "cstddef", + "cstdint", + "cstdio", + "cstdlib", + "cstring", + "ctime", + "cuchar", + "cwchar", + "cwctype", + ], +) diff --git a/libcxx/test/libcxx/numerics/rand/rand.req.urng/valid_real_type.verify.cpp b/libcxx/test/libcxx/numerics/rand/rand.req.urng/valid_real_type.verify.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6c6a109227df9f6b4e1a30d31ffa5d1fdc473630 --- /dev/null +++ b/libcxx/test/libcxx/numerics/rand/rand.req.urng/valid_real_type.verify.cpp @@ -0,0 +1,109 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +#include + +void test() { + { + std::uniform_real_distribution + baddist; //expected-error@*:* {{RealType must be a supported floating-point type}} + std::uniform_real_distribution okdist; + (void)baddist; + (void)okdist; + } + { + std::exponential_distribution + baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::exponential_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::gamma_distribution baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::gamma_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::weibull_distribution baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::weibull_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::extreme_value_distribution + baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::extreme_value_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::normal_distribution baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::normal_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::lognormal_distribution baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::lognormal_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::chi_squared_distribution + baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::chi_squared_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::cauchy_distribution baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::cauchy_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::fisher_f_distribution baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::fisher_f_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::student_t_distribution baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::student_t_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::piecewise_constant_distribution + baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::piecewise_constant_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::piecewise_linear_distribution + baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::piecewise_linear_distribution okdist; + (void)baddist; + (void)okdist; + } +} diff --git a/libcxx/test/libcxx/selftest/stdin-is-piped.sh.cpp b/libcxx/test/libcxx/selftest/stdin-is-piped.sh.cpp index 897e10e94783cf5602ca3bd19cf6cd674a0c935e..ffd10631c6a67d4ddd54e130be558bc8dd50bec5 100644 --- a/libcxx/test/libcxx/selftest/stdin-is-piped.sh.cpp +++ b/libcxx/test/libcxx/selftest/stdin-is-piped.sh.cpp @@ -8,8 +8,6 @@ // Make sure that the executor pipes standard input to the test-executable being run. -// XFAIL: LIBCXX-PICOLIBC-FIXME - // RUN: %{build} // RUN: echo "abc" | %{exec} %t.exe diff --git a/libcxx/test/lit.local.cfg b/libcxx/test/lit.local.cfg index 4116553b6f7a9aeb905037c5647554ac8957568f..1ee9086ee22e3c573fe66cef305888b2ba5b2a09 100644 --- a/libcxx/test/lit.local.cfg +++ b/libcxx/test/lit.local.cfg @@ -67,3 +67,17 @@ if ( "%{link_flags}", os.path.join(build, "libc++std.a"), ) + + config.substitutions = appendToSubstitution( + config.substitutions, + "%{compile_flags}", + "-fprebuilt-module-path=" + + os.path.join( + config.test_exec_root, "__config_module__/CMakeFiles/std.compat.dir" + ), + ) + config.substitutions = appendToSubstitution( + config.substitutions, + "%{link_flags}", + os.path.join(build, "libc++std.compat.a"), + ) diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.unique/ranges_unique_copy.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.unique/ranges_unique_copy.pass.cpp index b84600b92c2b291e103e25302479fa4163dbfaf1..a4cf97069c96a0f6d9e771f1458635d0d47e2978 100644 --- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.unique/ranges_unique_copy.pass.cpp +++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.unique/ranges_unique_copy.pass.cpp @@ -418,7 +418,7 @@ constexpr bool test() { assert(std::ranges::equal(out, expected)); assert(base(result.in) == in.end()); assert(base(result.out) == out.end()); - assert(numberOfComp == in.size() - 1); + assert(numberOfComp == static_cast(in.size() - 1)); assert(numberOfProj <= static_cast(2 * (in.size() - 1))); } // range overload @@ -434,7 +434,7 @@ constexpr bool test() { assert(std::ranges::equal(out, expected)); assert(base(result.in) == in.end()); assert(base(result.out) == out.end()); - assert(numberOfComp == in.size() - 1); + assert(numberOfComp == static_cast(in.size() - 1)); assert(numberOfProj <= static_cast(2 * (in.size() - 1))); } } diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/sort.heap/ranges_sort_heap.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/sort.heap/ranges_sort_heap.pass.cpp index 1153ed573d635f56e5a43f1308301714e2b38ddd..1e636ea9afac474b5ff466784eda697a79968a66 100644 --- a/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/sort.heap/ranges_sort_heap.pass.cpp +++ b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/sort.heap/ranges_sort_heap.pass.cpp @@ -238,6 +238,7 @@ void test_complexity() { const int debug_elements = std::min(100, n); // Multiplier 2 because of comp(a,b) comp(b, a) checks. const int debug_comparisons = 2 * (debug_elements + 1) * debug_elements; + (void)debug_comparisons; std::shuffle(first, last, g); std::make_heap(first, last, &MyInt::Comp); // The exact stats of our current implementation are recorded here. @@ -247,7 +248,6 @@ void test_complexity() { LIBCPP_ASSERT(stats.moved <= 2 * n + n * logn); #if _LIBCPP_HARDENING_MODE != _LIBCPP_HARDENING_MODE_DEBUG LIBCPP_ASSERT(stats.compared <= n * logn); - (void)debug_comparisons; #else LIBCPP_ASSERT(stats.compared <= 2 * n * logn + debug_comparisons); #endif diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/assign.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/assign.pass.cpp index 66f6c91b802cb0c852404002cbb74736b6a639cc..1561bd27d8d7e413ac2d93cfb4d2eb60e82c8e44 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/assign.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/assign.pass.cpp @@ -56,7 +56,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_strong.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_strong.pass.cpp index 711b49ddc23302872a4b11eab96f27eb2f708ddf..84ef7b70bc0aac58c3f298a16603281978f216be 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_strong.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_strong.pass.cpp @@ -220,7 +220,7 @@ void test() { int main(int, char**) { test(); test(); - // https://github.com/llvm/llvm-project/issues/47978 + // TODO https://github.com/llvm/llvm-project/issues/47978 // test(); return 0; diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_weak.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_weak.pass.cpp index ee1a00c284276c888fa7680f5c5aecd4de98d642..54ca29a0d427a5299b124b3cf799f5004396c8d9 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_weak.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_weak.pass.cpp @@ -68,7 +68,7 @@ void testBasic(MemoryOrder... memory_order) { assert(a.load() == T(1.2)); // bug - // https://github.com/llvm/llvm-project/issues/47978 + // TODO https://github.com/llvm/llvm-project/issues/47978 if constexpr (!std::same_as) { assert(expected == T(1.2)); } @@ -236,10 +236,8 @@ int main(int, char**) { test(); test(); -// https://github.com/llvm/llvm-project/issues/47978 -#ifndef TEST_COMPILER_CLANG - test(); -#endif + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/ctor.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/ctor.pass.cpp index aad63540847618fdd230975c622de9ef7d7630bf..92351d74f35c87f089bd4dae1372e48b8af8dcb9 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/ctor.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/ctor.pass.cpp @@ -56,7 +56,8 @@ constexpr void testOne() { constexpr bool test() { testOne(); testOne(); - testOne(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // testOne(); return true; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/exchange.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/exchange.pass.cpp index c0d1eb686db7a08f47fa3b51d1578d27c7b81d87..c365923c42d0ac91c80be6b0a28f521354b62e3b 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/exchange.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/exchange.pass.cpp @@ -6,9 +6,6 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// Clang's support for atomic operations on long double is broken. See https://github.com/llvm/llvm-project/issues/72893 -// XFAIL: target={{x86_64-.*}} && tsan -// XFAIL: target={{x86_64-.*}} && msan // XFAIL: !has-64-bit-atomics // UNSUPPORTED: !non-lockfree-atomics @@ -72,7 +69,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp index c31243053e69fba9cd804fe33e4eaf9e092588e9..d74502f52e1b37969aac4fb2aaf5f7e89f9afede 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp @@ -7,13 +7,12 @@ //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 // UNSUPPORTED: LIBCXX-AIX-FIXME -// Clang's support for atomic operations on long double is broken. See https://github.com/llvm/llvm-project/issues/72893 -// XFAIL: target={{x86_64-.*}} && tsan -// Hangs with msan. -// UNSUPPORTED: msan // XFAIL: !has-64-bit-atomics // UNSUPPORTED: !non-lockfree-atomics +// https://github.com/llvm/llvm-project/issues/72893 +// XFAIL: target={{x86_64-.*}} && tsan + // floating-point-type fetch_add(floating-point-type, // memory_order = memory_order::seq_cst) volatile noexcept; // floating-point-type fetch_add(floating-point-type, @@ -114,7 +113,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp index d26d21dec1b33466a8c72dedaf6e42e507393b8b..2b07f1fdbb5b5cb3bb80d2ebc48c81c913677226 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp @@ -7,13 +7,12 @@ //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 // UNSUPPORTED: LIBCXX-AIX-FIXME -// Clang's support for atomic operations on long double is broken. See https://github.com/llvm/llvm-project/issues/72893 -// XFAIL: target={{x86_64-.*}} && tsan -// Hangs with msan. -// UNSUPPORTED: msan // XFAIL: !has-64-bit-atomics // UNSUPPORTED: !non-lockfree-atomics +// https://github.com/llvm/llvm-project/issues/72893 +// XFAIL: target={{x86_64-.*}} && tsan + // floating-point-type fetch_sub(floating-point-type, // memory_order = memory_order::seq_cst) volatile noexcept; // floating-point-type fetch_sub(floating-point-type, @@ -115,7 +114,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/load.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/load.pass.cpp index b567af457dcaecd8e03d514794044cd40e885c4c..784fdb96c464defbd9f404ba8d47dc12375c9320 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/load.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/load.pass.cpp @@ -6,9 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// Clang's support for atomic operations on long double is broken. See https://github.com/llvm/llvm-project/issues/72893 // XFAIL: !has-64-bit-atomics -// XFAIL: target={{x86_64-.*}} && tsan // UNSUPPORTED: !non-lockfree-atomics // floating-point-type load(memory_order = memory_order::seq_cst) volatile noexcept; @@ -134,7 +132,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp index b5422c0362dc130df2657862e6cf6c8181682b2b..cf5af00cd4e8bc840b4ac18c927cb7878adc9b2a 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp @@ -55,7 +55,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_all.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_all.pass.cpp index b50c0ad9ee92efbf2d160084aec75c985178c2eb..d8bec9160c68f6c3c5cae0f4526227cdb90a7306 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_all.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_all.pass.cpp @@ -93,7 +93,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_one.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_one.pass.cpp index aee4af186b9ffe3f307dbcb6345ee7c3718cbf29..fd4ea44ff851d2ef2d5bc8b073fef42ac5bc02d4 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_one.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_one.pass.cpp @@ -77,7 +77,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.float.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.float.pass.cpp index 5d957ca68404a233f43c92e5014f9aafcbfef085..a1733276c0ec1723ba6f4126be51b3cdbe79fde8 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.float.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.float.pass.cpp @@ -52,7 +52,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp index 269eb819524f9d4583357176a5e1603644fada1c..22233b5a62ba931ec11482a41ca372ea23bf0034 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp @@ -6,11 +6,9 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: !has-64-bit-atomics // UNSUPPORTED: LIBCXX-AIX-FIXME +// XFAIL: !has-64-bit-atomics // UNSUPPORTED: !non-lockfree-atomics -// Hangs with msan. -// UNSUPPORTED: msan // floating-point-type operator-=(floating-point-type) volatile noexcept; // floating-point-type operator-=(floating-point-type) noexcept; @@ -98,7 +96,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp index 9a2298ea46d660b0532a3bb132d29d988cfda437..548c5ac11a265e800ef7188c5b02024fcf8f5274 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp @@ -6,11 +6,9 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: !has-64-bit-atomics // UNSUPPORTED: LIBCXX-AIX-FIXME +// XFAIL: !has-64-bit-atomics // UNSUPPORTED: !non-lockfree-atomics -// Hangs with msan. -// UNSUPPORTED: msan // floating-point-type operator+=(floating-point-type) volatile noexcept; // floating-point-type operator+=(floating-point-type) noexcept; @@ -98,7 +96,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/store.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/store.pass.cpp index b0888aac67dbbf537ef985bb4bf1d231e837dd2c..d232eea6ef014d2d82dc53ac3100bd91c2787a6a 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/store.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/store.pass.cpp @@ -6,9 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// Clang's support for atomic operations on long double is broken. See https://github.com/llvm/llvm-project/issues/72893 // XFAIL: !has-64-bit-atomics -// XFAIL: target={{x86_64-.*}} && tsan // UNSUPPORTED: !non-lockfree-atomics // void store(floating-point-type, memory_order = memory_order::seq_cst) volatile noexcept; @@ -109,7 +107,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/wait.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/wait.pass.cpp index 8e003a7e365da1a036282bee46e62a5ebf32ee04..ad70c133b99eb2da756190ec2f9a37e269c74aad 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/wait.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/wait.pass.cpp @@ -7,9 +7,6 @@ //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 // XFAIL: availability-synchronization_library-missing -// Clang's support for atomic operations on long double is broken. See https://github.com/llvm/llvm-project/issues/72893 -// XFAIL: target={{x86_64-.*}} && tsan -// XFAIL: target={{x86_64-.*}} && msan // XFAIL: !has-64-bit-atomics // UNSUPPORTED: !non-lockfree-atomics @@ -120,7 +117,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/containers/sequences/insert_range_sequence_containers.h b/libcxx/test/std/containers/sequences/insert_range_sequence_containers.h index b77b2510eae230af66e299cf071be52a48cacc52..352ee474cae7672712daea89f6ec00b70d3854e2 100644 --- a/libcxx/test/std/containers/sequences/insert_range_sequence_containers.h +++ b/libcxx/test/std/containers/sequences/insert_range_sequence_containers.h @@ -427,7 +427,8 @@ template <> constexpr TestCase FullContainer_End_LongRange { template constexpr void test_sequence_insert_range(Validate validate) { using T = typename Container::value_type; - auto get_pos = [](auto& c, auto& test_case) { return std::ranges::next(c.begin(), test_case.index); }; + using D = typename Container::difference_type; + auto get_pos = [](auto& c, auto& test_case) { return std::ranges::next(c.begin(), static_cast(test_case.index)); }; auto test = [&](auto& test_case) { Container c(test_case.initial.begin(), test_case.initial.end()); diff --git a/libcxx/test/std/containers/unord/unord.map/eq.different_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.map/eq.different_hash.pass.cpp index 52a806f04d72d6f4cf5ff37a11fe637abd186eae..94de7589eb060ee99ea3d2352aed94b857d8eac8 100644 --- a/libcxx/test/std/containers/unord/unord.map/eq.different_hash.pass.cpp +++ b/libcxx/test/std/containers/unord/unord.map/eq.different_hash.pass.cpp @@ -36,7 +36,7 @@ std::size_t hash_neg(T val) { } template std::size_t hash_scale(T val) { - return val << 1; + return static_cast(val << 1); } template std::size_t hash_even(T val) { @@ -57,7 +57,7 @@ std::size_t hash_neg(T* val) { } template std::size_t hash_scale(T* val) { - return *val << 1; + return static_cast(*val << 1); } template std::size_t hash_even(T* val) { diff --git a/libcxx/test/std/containers/unord/unord.multimap/eq.different_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/eq.different_hash.pass.cpp index 2644c96576446025dd1a38711415f19e35f2403f..0aafb401d42ca94b64a99cbf7641f64edee99d58 100644 --- a/libcxx/test/std/containers/unord/unord.multimap/eq.different_hash.pass.cpp +++ b/libcxx/test/std/containers/unord/unord.multimap/eq.different_hash.pass.cpp @@ -37,7 +37,7 @@ std::size_t hash_neg(T val) { } template std::size_t hash_scale(T val) { - return val << 1; + return static_cast(val << 1); } template std::size_t hash_even(T val) { @@ -58,7 +58,7 @@ std::size_t hash_neg(T* val) { } template std::size_t hash_scale(T* val) { - return *val << 1; + return static_cast(*val << 1); } template std::size_t hash_even(T* val) { diff --git a/libcxx/test/std/containers/unord/unord.multiset/eq.different_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/eq.different_hash.pass.cpp index 9f53e8d79e86652baad9bb4bd72216e388c4da4a..5b8f11e929279a57662c57c0675cf0e4623d2e2a 100644 --- a/libcxx/test/std/containers/unord/unord.multiset/eq.different_hash.pass.cpp +++ b/libcxx/test/std/containers/unord/unord.multiset/eq.different_hash.pass.cpp @@ -36,7 +36,7 @@ std::size_t hash_neg(T val) { } template std::size_t hash_scale(T val) { - return val << 1; + return static_cast(val << 1); } template std::size_t hash_even(T val) { @@ -57,7 +57,7 @@ std::size_t hash_neg(T* val) { } template std::size_t hash_scale(T* val) { - return *val << 1; + return static_cast(*val << 1); } template std::size_t hash_even(T* val) { diff --git a/libcxx/test/std/containers/unord/unord.set/eq.different_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.set/eq.different_hash.pass.cpp index a763c7fee623ab0d5ba31e1f11db625f8b0e2828..3cb4815a5bcb12135ccfce5ece288902338a1f44 100644 --- a/libcxx/test/std/containers/unord/unord.set/eq.different_hash.pass.cpp +++ b/libcxx/test/std/containers/unord/unord.set/eq.different_hash.pass.cpp @@ -36,7 +36,7 @@ std::size_t hash_neg(T val) { } template std::size_t hash_scale(T val) { - return val << 1; + return static_cast(val << 1); } template std::size_t hash_even(T val) { @@ -56,11 +56,11 @@ std::size_t hash_neg(T* val) { return std::numeric_limits::max() - *val; } template -size_t hash_scale(T* val) { - return *val << 1; +std::size_t hash_scale(T* val) { + return static_cast(*val << 1); } template -size_t hash_even(T* val) { +std::size_t hash_even(T* val) { return *val & 1 ? 1 : 0; } diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp index 01278e9076714a93294b6052d0e96c459ae4223d..30281a8d922d10d1c208f2b8187a1e09de70ce3f 100644 --- a/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp +++ b/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp @@ -53,10 +53,10 @@ constexpr void iterate_stride(M m, const std::array(M::extents_type::rank()) - 1 - static_cast(sizeof...(Args)); if constexpr (-1 == r) { ASSERT_NOEXCEPT(m(args...)); - size_t expected_val = [&](std::index_sequence) { + std::size_t expected_val = static_cast([&](std::index_sequence) { return ((args * strides[Pos]) + ... + 0); - }(std::make_index_sequence()); - assert(expected_val == static_cast(m(args...))); + }(std::make_index_sequence())); + assert(expected_val == static_cast(m(args...))); } else { for (typename M::index_type i = 0; i < m.extents().extent(r); i++) { iterate_stride(m, strides, i, args...); @@ -73,7 +73,7 @@ constexpr void test_iteration(std::array strides, Args... args) } constexpr bool test() { - constexpr size_t D = std::dynamic_extent; + constexpr std::size_t D = std::dynamic_extent; test_iteration>(std::array{}); test_iteration>(std::array{2}, 1); test_iteration>(std::array{3}, 7); @@ -102,7 +102,7 @@ constexpr bool test() { } constexpr bool test_large() { - constexpr size_t D = std::dynamic_extent; + constexpr std::size_t D = std::dynamic_extent; test_iteration>(std::array{2000, 2, 20, 200}, 7, 9, 10); test_iteration>(std::array{2000, 20, 20, 200}, 7, 10); return true; diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp index bbf8137a7cb85d54bd86e6e622bce6f9d4dcad4c..0e0c7667da307811f17a6444f2c22a4da52acf24 100644 --- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp +++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp @@ -65,7 +65,7 @@ test_mdspan_ctor_array(const H& handle, const M& map, const A&, std::array == (N == MDS::rank_dynamic())); diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp index 0a8918028d13c8121591b0a6a1505824b0f5529f..40e82db9863509d027e1e7181586e5cf642b8db3 100644 --- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp +++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp @@ -51,7 +51,7 @@ constexpr void test_mdspan_types(const H& handle, const M& map, const A&) { assert((H::move_counter() == 1)); } } - static_assert(!noexcept(MDS(handle, map.extents()))); + LIBCPP_STATIC_ASSERT(!noexcept(MDS(handle, map.extents()))); assert(m.extents() == map.extents()); if constexpr (std::equality_comparable) assert(m.data_handle() == handle); diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp index 75b7722b445e5a0bf16cd24bb72354fde6841382..fa65848ac69b53c04dc6ca23e13e101d77fecaa3 100644 --- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp +++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp @@ -48,7 +48,7 @@ constexpr void test_mdspan_types(const H& handle, const M& map, const A&) { assert((H::move_counter() == 1)); } } - static_assert(!noexcept(MDS(handle, map))); + LIBCPP_STATIC_ASSERT(!noexcept(MDS(handle, map))); assert(m.extents() == map.extents()); if constexpr (std::equality_comparable) assert(m.data_handle() == handle); diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp index c2210f1dba909069e8520897a35173b19d67e133..65d32f3d7a7f607fcd2f8544b4811198ae7d2dd6 100644 --- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp +++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp @@ -43,7 +43,7 @@ constexpr void test_mdspan_types(const H& handle, const M& map, const A& acc) { assert((H::move_counter() == 1)); } } - static_assert(!noexcept(MDS(handle, map, acc))); + LIBCPP_STATIC_ASSERT(!noexcept(MDS(handle, map, acc))); assert(m.extents() == map.extents()); if constexpr (std::equality_comparable) assert(m.data_handle() == handle); diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp index 980619795b5dfee9f03d8a4f45f23206b02db242..f4fb5e681d95f4d13f5f66866510a86e9caab25c 100644 --- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp +++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp @@ -65,7 +65,7 @@ test_mdspan_ctor_span(const H& handle, const M& map, const A&, std::span == (N == MDS::rank_dynamic())); diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp index 35534fa8795481467411c4e7ee2bee37bc2cb649..ba1fef1df6779a8f877b3e5695125de79f825ab8 100644 --- a/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp +++ b/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp @@ -27,11 +27,12 @@ // constexpr const data_handle_type& data_handle() const noexcept { return ptr_; } // constexpr const mapping_type& mapping() const noexcept { return map_; } // constexpr const accessor_type& accessor() const noexcept { return acc_; } -// static constexpr bool is_always_unique() +// /* per LWG-4021 "mdspan::is_always_meow() should be noexcept" */ +// static constexpr bool is_always_unique() noexcept // { return mapping_type::is_always_unique(); } -// static constexpr bool is_always_exhaustive() +// static constexpr bool is_always_exhaustive() noexcept // { return mapping_type::is_always_exhaustive(); } -// static constexpr bool is_always_strided() +// static constexpr bool is_always_strided() noexcept // { return mapping_type::is_always_strided(); } // // constexpr bool is_unique() const @@ -141,15 +142,16 @@ constexpr void test_mdspan_types(const H& handle, const M& map, const A& acc) { ASSERT_SAME_TYPE(decltype(m.is_unique()), bool); ASSERT_SAME_TYPE(decltype(m.is_exhaustive()), bool); ASSERT_SAME_TYPE(decltype(m.is_strided()), bool); - assert(!noexcept(MDS::is_always_unique())); - assert(!noexcept(MDS::is_always_exhaustive())); - assert(!noexcept(MDS::is_always_strided())); - assert(!noexcept(m.is_unique())); - assert(!noexcept(m.is_exhaustive())); - assert(!noexcept(m.is_strided())); - assert(MDS::is_always_unique() == M::is_always_unique()); - assert(MDS::is_always_exhaustive() == M::is_always_exhaustive()); - assert(MDS::is_always_strided() == M::is_always_strided()); + // per LWG-4021 "mdspan::is_always_meow() should be noexcept" + static_assert(noexcept(MDS::is_always_unique())); + static_assert(noexcept(MDS::is_always_exhaustive())); + static_assert(noexcept(MDS::is_always_strided())); + LIBCPP_STATIC_ASSERT(!noexcept(m.is_unique())); + LIBCPP_STATIC_ASSERT(!noexcept(m.is_exhaustive())); + LIBCPP_STATIC_ASSERT(!noexcept(m.is_strided())); + static_assert(MDS::is_always_unique() == M::is_always_unique()); + static_assert(MDS::is_always_exhaustive() == M::is_always_exhaustive()); + static_assert(MDS::is_always_strided() == M::is_always_strided()); assert(m.is_unique() == map.is_unique()); assert(m.is_exhaustive() == map.is_exhaustive()); assert(m.is_strided() == map.is_strided()); @@ -159,7 +161,7 @@ constexpr void test_mdspan_types(const H& handle, const M& map, const A& acc) { if (m.is_strided()) { for (typename MDS::rank_type r = 0; r < MDS::rank(); r++) { ASSERT_SAME_TYPE(decltype(m.stride(r)), typename MDS::index_type); - assert(!noexcept(m.stride(r))); + LIBCPP_STATIC_ASSERT(!noexcept(m.stride(r))); assert(m.stride(r) == map.stride(r)); } } diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/buffered_reads.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/buffered_reads.pass.cpp index d57b7c20a2da27254111f2e713d5411c31557c54..ecc11f4999ffa1ca38be5bcf2402392dcd140a4d 100644 --- a/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/buffered_reads.pass.cpp +++ b/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/buffered_reads.pass.cpp @@ -44,7 +44,7 @@ template void test_read(BufferPolicy policy, const std::vector& payload_sizes) { - std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), 0); + std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), std::streamsize{0}); std::vector data(total_size); for (std::size_t i = 0; i < data.size(); ++i) { data[i] = static_cast(i % (1 << 8 * sizeof(char))); @@ -99,7 +99,7 @@ void test_read(BufferPolicy policy, const std::vector& payload_ #ifndef TEST_HAS_NO_WIDE_CHARACTERS template void test_read_codecvt(BufferPolicy policy, const std::vector& payload_sizes) { - std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), 0); + std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), std::streamsize{0}); std::vector data(total_size); for (std::size_t i = 0; i < data.size(); ++i) { data[i] = static_cast(i); diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/buffered_writes.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/buffered_writes.pass.cpp index e7820739505106ad2bc2e6042add7e72b1e529b8..b5bbb0ca2ee4e759c1a41e64ec6953b246087e57 100644 --- a/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/buffered_writes.pass.cpp +++ b/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/buffered_writes.pass.cpp @@ -45,7 +45,7 @@ template void test_write(BufferPolicy policy, const std::vector& payload_sizes) { std::size_t previously_written = 0; - std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), 0); + std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), std::streamsize{0}); std::vector data(total_size); for (std::size_t i = 0; i < data.size(); ++i) { data[i] = static_cast(i % (1 << 8 * sizeof(char))); @@ -97,7 +97,7 @@ void test_write(BufferPolicy policy, const std::vector& payload template void test_write_codecvt(BufferPolicy policy, const std::vector& payload_sizes) { std::size_t previously_written = 0; - std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), 0); + std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), std::streamsize{0}); std::vector data(total_size); for (std::size_t i = 0; i < data.size(); ++i) { data[i] = static_cast(i); diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.assign/source.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.assign/source.pass.cpp index 893d09221fce2dcb446b038ea0bf728f9b5bc767..e969aa4e8d66aa33702dc3a4fff2e1faa217d01c 100644 --- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.assign/source.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.assign/source.pass.cpp @@ -230,7 +230,7 @@ void RunStringMoveTest(const fs::path::value_type* Expect) { assert(p == Expect); { // Signature test - LIBCPP_ONLY(ASSERT_NOEXCEPT(p = std::move(ss))); + LIBCPP_ASSERT_NOEXCEPT(p = std::move(ss)); } } diff --git a/libcxx/test/std/input.output/filesystems/fs.enum/enum.copy_options.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.enum/enum.copy_options.pass.cpp index 56322575fef213377c3151cc2e1dfce79b4e2bf7..4ef28ee01d8d02a3d1b1a2b74b517b668786408b 100644 --- a/libcxx/test/std/input.output/filesystems/fs.enum/enum.copy_options.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.enum/enum.copy_options.pass.cpp @@ -30,7 +30,7 @@ int main(int, char**) { typedef std::underlying_type::type UT; static_assert(!std::is_convertible::value, ""); - LIBCPP_ONLY(static_assert(std::is_same::value, "")); // Implementation detail + LIBCPP_STATIC_ASSERT(std::is_same::value, ""); // Implementation detail typedef check_bitmask_type BitmaskTester; assert(BitmaskTester::check()); diff --git a/libcxx/test/std/input.output/filesystems/fs.enum/enum.directory_options.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.enum/enum.directory_options.pass.cpp index 2597457ed747e1d2f62d5798a51e60be9e95caab..4480b1e4e3357cba42b5afed7cf5e8f2b27028d2 100644 --- a/libcxx/test/std/input.output/filesystems/fs.enum/enum.directory_options.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.enum/enum.directory_options.pass.cpp @@ -29,7 +29,7 @@ int main(int, char**) { // Check that E is a scoped enum by checking for conversions. typedef std::underlying_type::type UT; static_assert(!std::is_convertible::value, ""); - LIBCPP_ONLY(static_assert(std::is_same::value, "")); + LIBCPP_STATIC_ASSERT(std::is_same::value, ""); typedef check_bitmask_type BitmaskTester; assert(BitmaskTester::check()); diff --git a/libcxx/test/std/input.output/filesystems/fs.enum/enum.file_type.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.enum/enum.file_type.pass.cpp index 63db8892b0928c8bdefb75bf325de5335be93a3b..6062935126dd223fcbb25097826e857782b8e8be 100644 --- a/libcxx/test/std/input.output/filesystems/fs.enum/enum.file_type.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.enum/enum.file_type.pass.cpp @@ -29,7 +29,7 @@ int main(int, char**) { typedef std::underlying_type::type UT; static_assert(!std::is_convertible::value, ""); - LIBCPP_ONLY(static_assert(std::is_same::value, "")); // Implementation detail + LIBCPP_STATIC_ASSERT(std::is_same::value, ""); // Implementation detail // The standard doesn't specify the numeric values of the enum. LIBCPP_STATIC_ASSERT( diff --git a/libcxx/test/std/input.output/filesystems/fs.enum/enum.path.format.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.enum/enum.path.format.pass.cpp index 7f3022a3ce9bdd540eee0af686ac0bf83747e59e..ad0cdb092def8ff5d3e904d2c5d85893e5cddecb 100644 --- a/libcxx/test/std/input.output/filesystems/fs.enum/enum.path.format.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.enum/enum.path.format.pass.cpp @@ -25,9 +25,7 @@ int main(int, char**) { typedef fs::path::format E; static_assert(std::is_enum::value, ""); - typedef std::underlying_type::type UT; - - LIBCPP_ONLY(static_assert(std::is_same::value, "")); // Implementation detail + LIBCPP_STATIC_ASSERT(std::is_same::type, unsigned char>::value, ""); // Implementation detail static_assert( E::auto_format != E::native_format && diff --git a/libcxx/test/std/input.output/filesystems/fs.enum/enum.perm_options.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.enum/enum.perm_options.pass.cpp index afc64f6f00b4da1cdf6b97aec31dd4394184041d..0bdae4870238547b41a779ddb22d60a8478055c5 100644 --- a/libcxx/test/std/input.output/filesystems/fs.enum/enum.perm_options.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.enum/enum.perm_options.pass.cpp @@ -32,7 +32,7 @@ int main(int, char**) { typedef std::underlying_type::type UT; static_assert(!std::is_convertible::value, ""); - LIBCPP_ONLY(static_assert(std::is_same::value, "")); // Implementation detail + LIBCPP_STATIC_ASSERT(std::is_same::value, ""); // Implementation detail typedef check_bitmask_type BitmaskTester; assert(BitmaskTester::check()); diff --git a/libcxx/test/std/input.output/filesystems/fs.enum/enum.perms.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.enum/enum.perms.pass.cpp index 27aa5e29de20c45c1cdda705a1c9b842b36d294f..d1893847f01b43a2bf98ac60bf044f36a1b6d354 100644 --- a/libcxx/test/std/input.output/filesystems/fs.enum/enum.perms.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.enum/enum.perms.pass.cpp @@ -30,7 +30,7 @@ int main(int, char**) { typedef std::underlying_type::type UT; static_assert(!std::is_convertible::value, ""); - LIBCPP_ONLY(static_assert(std::is_same::value, "")); // Implementation detail + LIBCPP_STATIC_ASSERT(std::is_same::value, ""); // Implementation detail typedef check_bitmask_type BitmaskTester; assert(BitmaskTester::check()); diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.absolute/absolute.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.absolute/absolute.pass.cpp index 68fc09157c4b680a5185d3bddce0bd6642f7e59a..7a60d1ab29f4adf2f21a2d3e9eea1341291a239c 100644 --- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.absolute/absolute.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.absolute/absolute.pass.cpp @@ -51,7 +51,7 @@ static void basic_test() assert(!ec); assert(ret.is_absolute()); assert(PathEqIgnoreSep(ret, TC.expect)); - LIBCPP_ONLY(assert(PathEq(ret, TC.expect))); + LIBCPP_ASSERT(PathEq(ret, TC.expect)); } } diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.canonical/canonical.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.canonical/canonical.pass.cpp index 6e947a355a541393f874e81ae26636f2a1d03964..0098fe8ee698ef90db032313310c36f80c9bbd01 100644 --- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.canonical/canonical.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.canonical/canonical.pass.cpp @@ -104,7 +104,7 @@ static void test_exception_contains_paths() } catch (filesystem_error const& err) { assert(err.path1() == p); // libc++ provides the current path as the second path in the exception - LIBCPP_ONLY(assert(err.path2() == current_path())); + LIBCPP_ASSERT(err.path2() == current_path()); } fs::current_path(static_env.Dir); try { @@ -112,7 +112,7 @@ static void test_exception_contains_paths() assert(false); } catch (filesystem_error const& err) { assert(err.path1() == p); - LIBCPP_ONLY(assert(err.path2() == static_env.Dir)); + LIBCPP_ASSERT(err.path2() == static_env.Dir); } #endif } diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp index 0f5a7692bb70d6d0d32f7c9e8b74409882ffa2de..f009befa49c379c4edcc23218161ef11aae05cbc 100644 --- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp @@ -40,7 +40,7 @@ static void test_signatures() ASSERT_NOT_NOEXCEPT(fs::permissions(p, pr)); ASSERT_NOT_NOEXCEPT(fs::permissions(p, pr, opts)); ASSERT_NOEXCEPT(fs::permissions(p, pr, ec)); - LIBCPP_ONLY(ASSERT_NOT_NOEXCEPT(fs::permissions(p, pr, opts, ec))); + LIBCPP_ASSERT_NOT_NOEXCEPT(fs::permissions(p, pr, opts, ec)); } static void test_error_reporting() diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.temp_dir_path/temp_directory_path.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.temp_dir_path/temp_directory_path.pass.cpp index c0a98a62e6f9ad77998751cfdf7dca9bdc8c541b..aed87b73121d061ea054021c1ec5dd80c4ff41f7 100644 --- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.temp_dir_path/temp_directory_path.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.temp_dir_path/temp_directory_path.pass.cpp @@ -95,7 +95,7 @@ static void basic_tests() PutEnv(TC.name, dne); ec = GetTestEC(); ret = temp_directory_path(ec); - LIBCPP_ONLY(assert(ErrorIs(ec, expect_errc))); + LIBCPP_ASSERT(ErrorIs(ec, expect_errc)); assert(ec != GetTestEC()); assert(ec); assert(ret == ""); @@ -104,7 +104,7 @@ static void basic_tests() PutEnv(TC.name, file); ec = GetTestEC(); ret = temp_directory_path(ec); - LIBCPP_ONLY(assert(ErrorIs(ec, expect_errc))); + LIBCPP_ASSERT(ErrorIs(ec, expect_errc)); assert(ec != GetTestEC()); assert(ec); assert(ret == ""); diff --git a/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp b/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp index c9297318cd5d66710bd2f34d62be457712d931c2..f502616b677b7767afc4d6151fd8a7ae6a374fa8 100644 --- a/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp +++ b/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp @@ -69,7 +69,8 @@ static void test_vprint_unicode() { FILE* file = fmemopen(buffer.data(), buffer.size(), "wb"); assert(file); - std::vprint_unicode(file, "hello world{}", std::make_format_args('!')); + char c = '!'; + std::vprint_unicode(file, "hello world{}", std::make_format_args(c)); long pos = std::ftell(file); std::fclose(file); @@ -83,7 +84,8 @@ static void test_vprint_nonunicode() { FILE* file = fmemopen(buffer.data(), buffer.size(), "wb"); assert(file); - std::vprint_nonunicode(file, "hello world{}", std::make_format_args('!')); + char c = '!'; + std::vprint_nonunicode(file, "hello world{}", std::make_format_args(c)); long pos = std::ftell(file); std::fclose(file); diff --git a/libcxx/test/std/input.output/iostream.format/print.fun/vprint_nonunicode.sh.cpp b/libcxx/test/std/input.output/iostream.format/print.fun/vprint_nonunicode.sh.cpp index 63e1d2ad82b74e2135b0cd8fd1cbcf5bb07640f3..c1a690f559b11e35430cc3c80d6711d57116bb8d 100644 --- a/libcxx/test/std/input.output/iostream.format/print.fun/vprint_nonunicode.sh.cpp +++ b/libcxx/test/std/input.output/iostream.format/print.fun/vprint_nonunicode.sh.cpp @@ -35,8 +35,12 @@ int main(int, char**) { // The data is passed as-is so it does not depend on the encoding of the input. - std::vprint_nonunicode("{} {} ", std::make_format_args(1234, "一二三四")); - std::vprint_nonunicode("{} {}", std::make_format_args(true, nullptr)); + int i = 1234; + const char* s = "一二三四"; + bool b = true; + nullptr_t p = nullptr; + std::vprint_nonunicode("{} {} ", std::make_format_args(i, s)); + std::vprint_nonunicode("{} {}", std::make_format_args(b, p)); return 0; } diff --git a/libcxx/test/std/input.output/iostream.format/print.fun/vprint_unicode.sh.cpp b/libcxx/test/std/input.output/iostream.format/print.fun/vprint_unicode.sh.cpp index a9bcc33d2e014ac5f004e27bc4ab81a1c313ce71..198e71b55d9beb3ffdf6ee140e801000084bf886 100644 --- a/libcxx/test/std/input.output/iostream.format/print.fun/vprint_unicode.sh.cpp +++ b/libcxx/test/std/input.output/iostream.format/print.fun/vprint_unicode.sh.cpp @@ -35,8 +35,12 @@ int main(int, char**) { // The data is passed as-is so it does not depend on the encoding of the input. - std::vprint_unicode("{} {} ", std::make_format_args(1234, "一二三四")); - std::vprint_unicode("{} {}", std::make_format_args(true, nullptr)); + int i = 1234; + const char* s = "一二三四"; + bool b = true; + nullptr_t p = nullptr; + std::vprint_unicode("{} {} ", std::make_format_args(i, s)); + std::vprint_unicode("{} {}", std::make_format_args(b, p)); return 0; } diff --git a/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp b/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp index ad6cc192e2f48844b8ab96e667f080ce62aca7d0..ce6a7f37801e1042561a6c0216cc6cb2ecfeabef 100644 --- a/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp +++ b/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp @@ -8,7 +8,11 @@ // TODO: Investigate // UNSUPPORTED: LIBCXX-AIX-FIXME -// XFAIL: LIBCXX-PICOLIBC-FIXME + +// QEMU does not detect EOF, when reading from stdin +// "echo -n" suppresses any characters after the output and so the test hangs. +// https://gitlab.com/qemu-project/qemu/-/issues/1963 +// UNSUPPORTED: LIBCXX-PICOLIBC-FIXME // This test hangs on Android devices that lack shell_v2, which was added in // Android N (API 24). diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/member_swap_noexcept.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/member_swap_noexcept.pass.cpp index 0a0128e44658f84885f973548943201a1a9ba2da..63e2293c530bd00fffa05efb4225583c9692a862 100644 --- a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/member_swap_noexcept.pass.cpp +++ b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/member_swap_noexcept.pass.cpp @@ -84,7 +84,7 @@ static void test() { { std::basic_stringbuf, test_alloc_not_empty> buf1; std::basic_stringbuf, test_alloc_not_empty> buf; - static_assert(!noexcept(buf.swap(buf1))); + LIBCPP_STATIC_ASSERT(!noexcept(buf.swap(buf1))); } { std::basic_stringbuf, test_alloc_propagate_on_container_swap_not_empty> buf1; diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/nonmember_swap_noexcept.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/nonmember_swap_noexcept.pass.cpp index 4f41e3a4d716a2e679e248b732e33db13d61d8ae..a3706116f4090754504f78887bc4918c786f63e9 100644 --- a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/nonmember_swap_noexcept.pass.cpp +++ b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/nonmember_swap_noexcept.pass.cpp @@ -83,7 +83,7 @@ static void test() { { std::basic_stringbuf, test_alloc_not_empty> buf1; std::basic_stringbuf, test_alloc_not_empty> buf; - static_assert(!noexcept(swap(buf, buf1))); + LIBCPP_STATIC_ASSERT(!noexcept(swap(buf, buf1))); } { std::basic_stringbuf, test_alloc_propagate_on_container_swap_not_empty> buf1; diff --git a/libcxx/test/std/input.output/syncstream/syncbuf/syncstream.syncbuf.assign/swap.pass.cpp b/libcxx/test/std/input.output/syncstream/syncbuf/syncstream.syncbuf.assign/swap.pass.cpp index ba007da5a054a7b847b22aea089cd272fb7aed1c..a236bf4752a0762c5a7ef9c4a467524ff84a8eac 100644 --- a/libcxx/test/std/input.output/syncstream/syncbuf/syncstream.syncbuf.assign/swap.pass.cpp +++ b/libcxx/test/std/input.output/syncstream/syncbuf/syncstream.syncbuf.assign/swap.pass.cpp @@ -75,7 +75,8 @@ static void test_short_write_after_swap() { sync_buf2.sputn(expected.data(), expected.size()); sync_buf1.swap(sync_buf2); - expected.push_back(sync_buf1.sputc(CharT('B'))); + sync_buf1.sputc(CharT('B')); + expected.push_back(CharT('B')); sync_buf2.sputc(CharT('Z')); assert(sstr1.str().empty()); diff --git a/libcxx/test/std/iterators/iterator.requirements/iterator.cust/iterator.cust.move/iter_move.pass.cpp b/libcxx/test/std/iterators/iterator.requirements/iterator.cust/iterator.cust.move/iter_move.pass.cpp index 566638263e88755dae6207b2c9b07d0a1658f08e..9f293ff483cd861805b0ccacc07f57718d84f6db 100644 --- a/libcxx/test/std/iterators/iterator.requirements/iterator.cust/iterator.cust.move/iter_move.pass.cpp +++ b/libcxx/test/std/iterators/iterator.requirements/iterator.cust/iterator.cust.move/iter_move.pass.cpp @@ -158,15 +158,15 @@ constexpr bool test() { auto unscoped = check_unqualified_lookup::unscoped_enum::a; assert(std::ranges::iter_move(unscoped) == check_unqualified_lookup::unscoped_enum::a); - assert(!noexcept(std::ranges::iter_move(unscoped))); + static_assert(!noexcept(std::ranges::iter_move(unscoped))); auto scoped = check_unqualified_lookup::scoped_enum::a; assert(std::ranges::iter_move(scoped) == nullptr); - assert(noexcept(std::ranges::iter_move(scoped))); + static_assert(noexcept(std::ranges::iter_move(scoped))); auto some_union = check_unqualified_lookup::some_union{0}; assert(std::ranges::iter_move(some_union) == 0); - assert(!noexcept(std::ranges::iter_move(some_union))); + static_assert(!noexcept(std::ranges::iter_move(some_union))); // Check noexcept-correctness static_assert(noexcept(std::ranges::iter_move(std::declval>()))); diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.indirect.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.indirect.pass.cpp index 172b6cc2f2944ad325a8c8ccdd41dbd9592cfd8e..f6f586a2b54786195afef83be73fc57d70ea9d0f 100644 --- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.indirect.pass.cpp +++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.indirect.pass.cpp @@ -29,7 +29,9 @@ TEST_WORKAROUND_BUG_109234844_WEAK void* operator new(std::size_t s) TEST_THROW_SPEC(std::bad_alloc) { ++new_called; void* ret = std::malloc(s); - if (!ret) std::abort(); // placate MSVC's unchecked malloc warning + if (!ret) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } return ret; } diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.pass.cpp index e352c00b4d0af943376a215033e92529e8b68155..29e739d8515fd6e0b21bfaea1991170e1454f009 100644 --- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.pass.cpp +++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.pass.cpp @@ -27,7 +27,9 @@ int delete_called = 0; void* operator new[](std::size_t s) TEST_THROW_SPEC(std::bad_alloc) { ++new_called; void* ret = std::malloc(s); - if (!ret) std::abort(); // placate MSVC's unchecked malloc warning + if (!ret) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } return ret; } diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_nothrow.replace.indirect.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_nothrow.replace.indirect.pass.cpp index 8ad0292dcb5ca4becbe45521c98fd78e61ee7dde..b26eec0324af6d3302038bac3d27c5523000001b 100644 --- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_nothrow.replace.indirect.pass.cpp +++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_nothrow.replace.indirect.pass.cpp @@ -33,7 +33,9 @@ TEST_WORKAROUND_BUG_109234844_WEAK void* operator new[](std::size_t s) TEST_THROW_SPEC(std::bad_alloc) { ++new_called; void* ret = std::malloc(s); - if (!ret) std::abort(); // placate MSVC's unchecked malloc warning + if (!ret) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } return ret; } diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.replace.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.replace.pass.cpp index a03313e5872ef3b8baf18b7b497b0dc82b2f7c3d..ab1cf5ea4644803fbc0fe0067f71cbe5aa208f16 100644 --- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.replace.pass.cpp +++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.replace.pass.cpp @@ -25,7 +25,9 @@ int delete_called = 0; void* operator new(std::size_t s) TEST_THROW_SPEC(std::bad_alloc) { ++new_called; void* ret = std::malloc(s); - if (!ret) std::abort(); // placate MSVC's unchecked malloc warning + if (!ret) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } return ret; } diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align_nothrow.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align_nothrow.pass.cpp index 4e5d36cd7c6dfcd8f3b7b1093bc8d9dc28e689fb..1c575729678d5cff949b74337ea41d54b16799e7 100644 --- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align_nothrow.pass.cpp +++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align_nothrow.pass.cpp @@ -60,8 +60,9 @@ int main(int, char**) { assert(reinterpret_cast(x) % alignof(TrackLifetimeOverAligned) == 0); assert(info.address_constructed == x); + const auto old_x = x; delete x; - assert(info.address_destroyed == x); + assert(info.address_destroyed == old_x); } return 0; diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.pass.cpp index 398de0068aba1b29c6f77c3914045ed2cd12be9e..56ae8df43f66fbf066be2ae337f66e53a13e6b9c 100644 --- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.pass.cpp +++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.pass.cpp @@ -50,8 +50,9 @@ int main(int, char**) { assert(x != nullptr); assert(info.address_constructed == x); + const auto old_x = x; delete x; - assert(info.address_destroyed == x); + assert(info.address_destroyed == old_x); } return 0; diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.replace.indirect.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.replace.indirect.pass.cpp index 2ae0dfa4f1abc41b0c3d46086d4d124ec80ecd38..35a601339ddd3642b420bb4a5f458aec64e20583 100644 --- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.replace.indirect.pass.cpp +++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.replace.indirect.pass.cpp @@ -28,7 +28,9 @@ TEST_WORKAROUND_BUG_109234844_WEAK void* operator new(std::size_t s) TEST_THROW_SPEC(std::bad_alloc) { ++new_called; void* ret = std::malloc(s); - if (!ret) std::abort(); // placate MSVC's unchecked malloc warning + if (!ret) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } return ret; } diff --git a/libcxx/test/std/modules/std.compat.pass.cpp b/libcxx/test/std/modules/std.compat.pass.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a33ed3b6b64533aafab69d80d83e9ae8dffe821f --- /dev/null +++ b/libcxx/test/std/modules/std.compat.pass.cpp @@ -0,0 +1,18 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17 + +// UNSUPPORTED: libcpp-has-no-std-modules +// UNSUPPORTED: clang-modules-build + +// A minimal test to validate import works. + +import std.compat; + +int main(int, char**) { return !(::strlen("Hello modular world") == 19); } diff --git a/libcxx/test/std/ranges/range.adaptors/range.elements/general.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.elements/general.pass.cpp index 78792ae54bdbfcba2bbb981c0104bf69636add68..d5318ced73dcd9848c46dedab41dfd813f0e4bf5 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.elements/general.pass.cpp +++ b/libcxx/test/std/ranges/range.adaptors/range.elements/general.pass.cpp @@ -70,7 +70,7 @@ int main(int, char**) { // tuple { - std::tuple tps[] = {{1}, {2}, {3}}; + std::tuple tps[] = {{short{1}}, {short{2}}, {short{3}}}; auto ev = tps | std::views::elements<0>; auto expected = {1, 2, 3}; assert(std::ranges::equal(ev, expected)); diff --git a/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/base.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/base.pass.cpp index 3729c8e543113120e38fde78ee25a5d41d3e9c6e..79c4e1f41837296d9647146d3b41497914a0a9e6 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/base.pass.cpp +++ b/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/base.pass.cpp @@ -33,7 +33,7 @@ using ElementsIter = std::ranges::iterator_t); static_assert(IsBaseNoexcept); static_assert(IsBaseNoexcept); -static_assert(!IsBaseNoexcept); +LIBCPP_STATIC_ASSERT(!IsBaseNoexcept); constexpr bool test() { std::tuple t{5}; diff --git a/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/deref.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/deref.pass.cpp index d87a3e53392036b2ff3e0aa94dc7acdf586cf198..f88091f42699e10db7a42a7a2bb18a1f70537cad 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/deref.pass.cpp +++ b/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/deref.pass.cpp @@ -50,7 +50,7 @@ constexpr void testValue(T t) { constexpr bool test() { // test tuple { - std::tuple ts[] = {{1, 2, 3}, {4, 5, 6}}; + std::tuple ts[] = {{1, short{2}, 3}, {4, short{5}, 6}}; testReference<0>(ts); testReference<1>(ts); testReference<2>(ts); @@ -61,7 +61,7 @@ constexpr bool test() { // test pair { - std::pair ps[] = {{1, 2}, {4, 5}}; + std::pair ps[] = {{1, short{2}}, {4, short{5}}}; testReference<0>(ps); testReference<1>(ps); testValue<0>(ps[0]); diff --git a/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/member_types.compile.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/member_types.compile.pass.cpp index 9a76c2fcb70c2439159977e5735fab9bf9f3cab0..70d49c1304b54199b872cb527cfcb7bc9456f8c8 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/member_types.compile.pass.cpp +++ b/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/member_types.compile.pass.cpp @@ -64,9 +64,7 @@ static_assert(std::same_as*>>::iterator_category, // std::random_access_iterator_tag>); -using Generator = decltype(std::views::iota(0, 1) | std::views::transform([](int) { - return std::pair{1, 1}; - })); +using Generator = decltype(std::views::iota(0, 1) | std::views::transform([](int) { return std::pair{1, short{1}}; })); static_assert(std::ranges::random_access_range); static_assert(std::same_as::iterator_category, // diff --git a/libcxx/test/std/ranges/range.utility/range.utility.conv/container.h b/libcxx/test/std/ranges/range.utility/range.utility.conv/container.h index fafccacd456d5b89b4b62f9c3edb4a125cab6eb1..ca89e3757affc786623eec49985150b059ec0dba 100644 --- a/libcxx/test/std/ranges/range.utility/range.utility.conv/container.h +++ b/libcxx/test/std/ranges/range.utility/range.utility.conv/container.h @@ -38,7 +38,7 @@ struct Container { constexpr explicit Container(std::ranges::input_range auto&& in) requires(Rank >= CtrChoice::DirectCtr) - : ctr_choice(CtrChoice::DirectCtr), size_(std::ranges::size(in)) { + : ctr_choice(CtrChoice::DirectCtr), size_(static_cast(std::ranges::size(in))) { std::ranges::copy(in, begin()); } @@ -54,7 +54,7 @@ struct Container { constexpr Container(std::from_range_t, std::ranges::input_range auto&& in) requires(Rank >= CtrChoice::FromRangeT) - : ctr_choice(CtrChoice::FromRangeT), size_(std::ranges::size(in)) { + : ctr_choice(CtrChoice::FromRangeT), size_(static_cast(std::ranges::size(in))) { std::ranges::copy(in, begin()); } @@ -70,7 +70,7 @@ struct Container { template constexpr Container(Iter b, Iter e) requires(Rank >= CtrChoice::BeginEndPair) - : ctr_choice(CtrChoice::BeginEndPair), size_(e - b) { + : ctr_choice(CtrChoice::BeginEndPair), size_(static_cast(e - b)) { std::ranges::copy(b, e, begin()); } diff --git a/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/F.pass.cpp b/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/F.pass.cpp index fc3d5c12227464458e693611c3c0d52d401a6d16..1c520652ba2297dec9e96e983b46223d4cade3a9 100644 --- a/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/F.pass.cpp +++ b/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/F.pass.cpp @@ -36,7 +36,9 @@ void* operator new(std::size_t s) TEST_THROW_SPEC(std::bad_alloc) } while (!throw_one.compare_exchange_weak(expected, expected - 1)); ++outstanding_new; void* ret = std::malloc(s); - if (!ret) std::abort(); // placate MSVC's unchecked malloc warning + if (!ret) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } return ret; } diff --git a/libcxx/test/std/time/time.syn/formatter_tests.h b/libcxx/test/std/time/time.syn/formatter_tests.h index 42c176b3b47e5bff068f12d3fe4e7851d064cf92..1b343b5c8711b68b481ae8e1bbebdbf30d6728a1 100644 --- a/libcxx/test/std/time/time.syn/formatter_tests.h +++ b/libcxx/test/std/time/time.syn/formatter_tests.h @@ -8,6 +8,8 @@ #ifndef TEST_STD_TIME_TIME_SYN_FORMATTER_TESTS_H #define TEST_STD_TIME_TIME_SYN_FORMATTER_TESTS_H +#include "assert_macros.h" +#include "concat_macros.h" #include "make_string.h" #include "string_literal.h" #include "test_format_string.h" @@ -34,11 +36,9 @@ using format_context = std::format_context; template void check(std::basic_string_view expected, test_format_string fmt, Args&&... args) { std::basic_string out = std::format(fmt, std::forward(args)...); - if constexpr (std::same_as) - if (out != expected) - std::cerr << "\nFormat string " << fmt.get() << "\nExpected output " << expected << "\nActual output " << out - << '\n'; - assert(out == expected); + TEST_REQUIRE(out == expected, + TEST_WRITE_CONCATENATED( + "\nFormat string ", fmt.get(), "\nExpected output ", expected, "\nActual output ", out, '\n')); } template @@ -47,38 +47,24 @@ void check(const std::locale& loc, test_format_string fmt, Args&&... args) { std::basic_string out = std::format(loc, fmt, std::forward(args)...); - if constexpr (std::same_as) - if (out != expected) - std::cerr << "\nFormat string " << fmt.get() << "\nExpected output " << expected << "\nActual output " << out - << '\n'; - assert(out == expected); + TEST_REQUIRE(out == expected, + TEST_WRITE_CONCATENATED( + "\nFormat string ", fmt.get(), "\nExpected output ", expected, "\nActual output ", out, '\n')); } template void check_exception([[maybe_unused]] std::string_view what, [[maybe_unused]] std::basic_string_view fmt, [[maybe_unused]] const Args&... args) { -#ifndef TEST_HAS_NO_EXCEPTIONS - try { - TEST_IGNORE_NODISCARD std::vformat(fmt, std::make_format_args>(args...)); - if constexpr (std::same_as) - std::cerr << "\nFormat string " << fmt << "\nDidn't throw an exception.\n"; - assert(false); - } catch (const std::format_error& e) { -# if defined(_LIBCPP_VERSION) - if constexpr (std::same_as) - if (e.what() != what) - std::cerr << "\nFormat string " << fmt << "\nExpected exception " << what << "\nActual exception " - << e.what() << '\n'; - assert(e.what() == what); -# else - (void)what; - (void)e; -# endif - return; - } - assert(false); -#endif + TEST_VALIDATE_EXCEPTION( + std::format_error, + [&]([[maybe_unused]] const std::format_error& e) { + TEST_LIBCPP_REQUIRE( + e.what() == what, + TEST_WRITE_CONCATENATED( + "\nFormat string ", fmt, "\nExpected exception ", what, "\nActual exception ", e.what(), '\n')); + }, + TEST_IGNORE_NODISCARD std::vformat(fmt, std::make_format_args>(args...))); } template diff --git a/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.pass.cpp b/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.pass.cpp index 72184bbd3920a7461b28048c3960ed20d3726dfe..62fd0f25ae3a9550ad92a5d82a8fe7cefe36e22d 100644 --- a/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.pass.cpp +++ b/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.pass.cpp @@ -10,7 +10,7 @@ // // template -// format-arg-store make_format_args(const Args&... args); +// format-arg-store make_format_args(Args&... args); #include #include @@ -20,8 +20,19 @@ #include "test_basic_format_arg.h" #include "test_macros.h" +template +concept can_make_format_args = requires(Args&&... args) { std::make_format_args(std::forward(args)...); }; + +static_assert(can_make_format_args); +static_assert(!can_make_format_args); +static_assert(!can_make_format_args); + int main(int, char**) { - [[maybe_unused]] auto store = std::make_format_args(42, nullptr, false, 'x'); + int i = 1; + char c = 'c'; + nullptr_t p = nullptr; + bool b = false; + [[maybe_unused]] auto store = std::make_format_args(i, p, b, c); LIBCPP_STATIC_ASSERT( std::same_as>); diff --git a/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.sh.cpp b/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.sh.cpp index 95a94b0bff5d07bce2ef532302885755ba8efaa4..2d5ee8349b749ff7425bd0f19da509aebb02574f 100644 --- a/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.sh.cpp +++ b/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.sh.cpp @@ -24,6 +24,7 @@ #include "test_macros.h" void test() { + char c = 'c'; TEST_IGNORE_NODISCARD - std::make_format_args>, wchar_t>>('c'); + std::make_format_args>, wchar_t>>(c); } diff --git a/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_wformat_args.pass.cpp b/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_wformat_args.pass.cpp index 22c6f031efc6f71b6f4e9f83e82f7afaf817934a..73c4395a4a630809b1edc3a50d707fd3c5608ef1 100644 --- a/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_wformat_args.pass.cpp +++ b/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_wformat_args.pass.cpp @@ -12,7 +12,7 @@ // template // format-arg-store -// make_wformat_args(const Args&... args); +// make_wformat_args(Args&... args); #include #include @@ -20,8 +20,19 @@ #include "test_basic_format_arg.h" #include "test_macros.h" +template +concept can_make_wformat_args = requires(Args&&... args) { std::make_wformat_args(std::forward(args)...); }; + +static_assert(can_make_wformat_args); +static_assert(!can_make_wformat_args); +static_assert(!can_make_wformat_args); + int main(int, char**) { - [[maybe_unused]] auto store = std::make_wformat_args(42, nullptr, false, 'x'); + int i = 1; + char c = 'c'; + nullptr_t p = nullptr; + bool b = false; + [[maybe_unused]] auto store = std::make_wformat_args(i, p, b, c); LIBCPP_STATIC_ASSERT( std::same_as>); diff --git a/libcxx/test/std/utilities/format/format.arguments/format.args/ctad.compile.pass.cpp b/libcxx/test/std/utilities/format/format.arguments/format.args/ctad.compile.pass.cpp index 7cadd4e76c74e0f15363bd202230f664bbda7171..b87b5c774ef7e7b2e5fbfd7d214f797f40f33c1f 100644 --- a/libcxx/test/std/utilities/format/format.arguments/format.args/ctad.compile.pass.cpp +++ b/libcxx/test/std/utilities/format/format.arguments/format.args/ctad.compile.pass.cpp @@ -18,12 +18,13 @@ #include "test_macros.h" void test() { + int i = 1; // Note the Standard way to create a format-arg-store is by using make_format_args. - static_assert(std::same_as>); #ifndef TEST_HAS_NO_WIDE_CHARACTERS - static_assert(std::same_as>); #endif diff --git a/libcxx/test/std/utilities/format/format.arguments/format.args/ctor.pass.cpp b/libcxx/test/std/utilities/format/format.arguments/format.args/ctor.pass.cpp index c2d2d19d978bbd1eecda4184b9c35ca6aadbbd9d..c0575c545bde31f5b0076f2908037203fb99146d 100644 --- a/libcxx/test/std/utilities/format/format.arguments/format.args/ctor.pass.cpp +++ b/libcxx/test/std/utilities/format/format.arguments/format.args/ctor.pass.cpp @@ -20,6 +20,9 @@ template void test() { + int i = 1; + char c = 'c'; + nullptr_t p = nullptr; using Context = std::basic_format_context; { ASSERT_NOEXCEPT(std::basic_format_args{}); @@ -28,14 +31,14 @@ void test() { assert(!format_args.get(0)); } { - auto store = std::make_format_args(1); + auto store = std::make_format_args(i); ASSERT_NOEXCEPT(std::basic_format_args{store}); std::basic_format_args format_args{store}; assert(format_args.get(0)); assert(!format_args.get(1)); } { - auto store = std::make_format_args(1, 'c'); + auto store = std::make_format_args(i, c); ASSERT_NOEXCEPT(std::basic_format_args{store}); std::basic_format_args format_args{store}; assert(format_args.get(0)); @@ -43,7 +46,7 @@ void test() { assert(!format_args.get(2)); } { - auto store = std::make_format_args(1, 'c', nullptr); + auto store = std::make_format_args(i, c, p); ASSERT_NOEXCEPT(std::basic_format_args{store}); std::basic_format_args format_args{store}; assert(format_args.get(0)); diff --git a/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/arg.pass.cpp b/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/arg.pass.cpp index 153f8bbaa4b37f8a54fb5318e8065c7d6f6ee61c..824813d33a519a37d7ccd961e9f99f39c65c4c76 100644 --- a/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/arg.pass.cpp +++ b/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/arg.pass.cpp @@ -23,9 +23,11 @@ template void test() { + bool b = true; + CharT c = CharT('a'); + int a = 42; std::basic_string string = MAKE_STRING(CharT, "string"); - auto store = std::make_format_args>( - true, CharT('a'), 42, string); + auto store = std::make_format_args>(b, c, a, string); std::basic_format_args args = store; std::basic_string output; diff --git a/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/ctor.pass.cpp b/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/ctor.pass.cpp index 4384f7b0fe68cfa2707cbf71438a9094a5e69fe6..40720105060f0d57b4f58542d9489a718bea6cf0 100644 --- a/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/ctor.pass.cpp +++ b/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/ctor.pass.cpp @@ -36,11 +36,14 @@ template void test() { + int a = 42; + bool b = true; + CharT c = CharT('a'); std::basic_string string = MAKE_STRING(CharT, "string"); // The type of the object is an exposition only type. The temporary is needed // to extend the lifetime of the object since args stores a pointer to the // data in this object. - auto format_arg_store = std::make_format_args>(true, CharT('a'), 42, string); + auto format_arg_store = std::make_format_args>(b, c, a, string); std::basic_format_args args = format_arg_store; { diff --git a/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/locale.pass.cpp b/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/locale.pass.cpp index 5533fe1b9f280b54fd3086f0a4f74238b3ba6ebe..14bdc1426098e1ffa2eeb66f0c951c3df55eb955 100644 --- a/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/locale.pass.cpp +++ b/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/locale.pass.cpp @@ -34,7 +34,10 @@ void test() { // The type of the object is an exposition only type. The temporary is needed // to extend the lifetime of the object since args stores a pointer to the // data in this object. - auto format_arg_store = std::make_format_args>(true, CharT('a'), 42, string); + int a = 42; + bool b = true; + CharT c = CharT('a'); + auto format_arg_store = std::make_format_args>(b, c, a, string); std::basic_format_args args = format_arg_store; { diff --git a/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.string.pass.cpp b/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.string.pass.cpp index 292de0cdd8450cfc3c3153866a806b1917bc4286..e99deb2db5530071c41230a991c7f6a20a1ea093 100644 --- a/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.string.pass.cpp +++ b/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.string.pass.cpp @@ -53,8 +53,7 @@ void test(StringT expected, StringViewT fmt, StringT a, std::size_t offset) { using FormatCtxT = std::basic_format_context; ArgumentT arg = a; - FormatCtxT format_ctx = test_format_context_create( - out, std::make_format_args(std::forward(arg))); + FormatCtxT format_ctx = test_format_context_create(out, std::make_format_args(arg)); formatter.format(arg, format_ctx); assert(result == expected); } diff --git a/libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp b/libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp index 5c6f86f742baafbcce6ae47b4cca21899abe040e..23b26722d7be7fdc5cb2e75f29cf6725c1f049ab 100644 --- a/libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp +++ b/libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp @@ -515,7 +515,7 @@ static void test_ill_formed_utf8() { } #ifndef TEST_HAS_NO_WIDE_CHARACTERS -# ifdef _LIBCPP_SHORT_WCHAR +# ifdef TEST_SHORT_WCHAR static void test_ill_formed_utf16() { using namespace std::literals; @@ -543,7 +543,7 @@ static void test_ill_formed_utf16() { L"\xdbff" "a"); } -# else // _LIBCPP_SHORT_WCHAR +# else // TEST_SHORT_WCHAR static void test_ill_formed_utf32() { using namespace std::literals; @@ -552,7 +552,7 @@ static void test_ill_formed_utf32() { test_format(LR"("\x{ffffffff}")"sv, L"{:?}", L"\xffffffff"); // largest encoded code point } -# endif // _LIBCPP_SHORT_WCHAR +# endif // TEST_SHORT_WCHAR #endif // TEST_HAS_NO_WIDE_CHARACTERS int main(int, char**) { @@ -563,11 +563,11 @@ int main(int, char**) { test_ill_formed_utf8(); #ifndef TEST_HAS_NO_WIDE_CHARACTERS -# ifdef _LIBCPP_SHORT_WCHAR +# ifdef TEST_SHORT_WCHAR test_ill_formed_utf16(); -# else // _LIBCPP_SHORT_WCHAR +# else // TEST_SHORT_WCHAR test_ill_formed_utf32(); -# endif // _LIBCPP_SHORT_WCHAR +# endif // TEST_SHORT_WCHAR #endif // TEST_HAS_NO_WIDE_CHARACTERS return 0; diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.const/type_conv_ctor.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.const/type_conv_ctor.pass.cpp index 2759b921fabe0cce886c6abd7f329e42fcf41cd1..48c62ae7d45bbb7d3352b303c6de766f080d9ca1 100644 --- a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.const/type_conv_ctor.pass.cpp +++ b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.const/type_conv_ctor.pass.cpp @@ -37,8 +37,9 @@ struct convertible_from_int { void meow(std::reference_wrapper) {} void meow(convertible_from_int) {} -int main(int, char**) -{ +std::reference_wrapper purr(); + +int main(int, char**) { { convertible_to_int_ref t; std::reference_wrapper r(t); @@ -54,21 +55,18 @@ int main(int, char**) ASSERT_NOEXCEPT(Ref(nothrow_convertible())); ASSERT_NOT_NOEXCEPT(Ref(nothrow_convertible())); } - { - meow(0); - } - { - extern std::reference_wrapper purr(); - ASSERT_SAME_TYPE(decltype(true ? purr() : 0), int); - } + meow(0); + ASSERT_SAME_TYPE(decltype(true ? purr() : 0), int); #if TEST_STD_VER > 14 { int i = 0; std::reference_wrapper ri(i); - static_assert((std::is_same>::value), "" ); + static_assert((std::is_same>::value), ""); + } + { const int j = 0; std::reference_wrapper rj(j); - static_assert((std::is_same>::value), "" ); + static_assert((std::is_same>::value), ""); } #endif diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared_for_overwrite.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared_for_overwrite.pass.cpp index 27ff3cd563740c906df4ee113ac02ab0d64eec3c..e6e063304453a5f090dfbaf29d36e879d117aa72 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared_for_overwrite.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared_for_overwrite.pass.cpp @@ -156,7 +156,7 @@ void testAllocatorOperationsCalled() { template struct AllocatorWithPattern { - constexpr static char pattern = 0xDE; + constexpr static char pattern = static_cast(0xDE); using value_type = T; diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared_for_overwrite.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared_for_overwrite.pass.cpp index 21e1786f015882ae8123db4c7405740a498310a6..96363060a7beb4b6cc88989fe29194f8d14c0ecc 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared_for_overwrite.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared_for_overwrite.pass.cpp @@ -56,10 +56,13 @@ static_assert(!HasMakeSharedForOverwrite); static_assert(!HasMakeSharedForOverwrite); static_assert(!HasMakeSharedForOverwrite); -constexpr char pattern = 0xDE; +constexpr char pattern = static_cast(0xDE); void* operator new(std::size_t count) { void* ptr = std::malloc(count); + if (!ptr) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } for (std::size_t i = 0; i < count; ++i) { *(reinterpret_cast(ptr) + i) = pattern; } diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.create/make_unique_for_overwrite.default_init.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.create/make_unique_for_overwrite.default_init.pass.cpp index 8011a37be08ecd33f9444aa0637d3069b7aea660..3ac0ce962cfa59b5ede4d9742d45cf256ec90ad4 100644 --- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.create/make_unique_for_overwrite.default_init.pass.cpp +++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.create/make_unique_for_overwrite.default_init.pass.cpp @@ -23,10 +23,13 @@ #include #include -constexpr char pattern = 0xDE; +constexpr char pattern = static_cast(0xDE); void* operator new(std::size_t count) { void* ptr = std::malloc(count); + if (!ptr) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } for (std::size_t i = 0; i < count; ++i) { *(reinterpret_cast(ptr) + i) = pattern; } diff --git a/libcxx/test/support/allocators.h b/libcxx/test/support/allocators.h index 2b987ad872783ec4a771a3edb1771b9b178c02d9..02436fd9c35ef1b19a8f6cb0a6f098999d9f0594 100644 --- a/libcxx/test/support/allocators.h +++ b/libcxx/test/support/allocators.h @@ -209,7 +209,7 @@ public: : id_(id), copy_assigned_into_(copy_assigned_into) {} template - MaybePOCCAAllocator(const MaybePOCCAAllocator& that) + TEST_CONSTEXPR MaybePOCCAAllocator(const MaybePOCCAAllocator& that) : id_(that.id_), copy_assigned_into_(that.copy_assigned_into_) {} MaybePOCCAAllocator(const MaybePOCCAAllocator&) = default; diff --git a/libcxx/test/support/concat_macros.h b/libcxx/test/support/concat_macros.h index 8d80a8c9f69ca28e600491f1cb848208b6ebdbfb..d7340b8faf6e562aa5e30de55a28adb5c4569417 100644 --- a/libcxx/test/support/concat_macros.h +++ b/libcxx/test/support/concat_macros.h @@ -16,15 +16,154 @@ #include "test_macros.h" #ifndef TEST_HAS_NO_LOCALIZATION +# include +# include # include #endif #if TEST_STD_VER > 17 # ifndef TEST_HAS_NO_LOCALIZATION + +[[nodiscard]] constexpr bool test_is_high_surrogate(char32_t value) { return value >= 0xd800 && value <= 0xdbff; } + +[[nodiscard]] constexpr bool test_is_low_surrogate(char32_t value) { return value >= 0xdc00 && value <= 0xdfff; } + +[[nodiscard]] constexpr bool test_is_surrogate(char32_t value) { return value >= 0xd800 && value <= 0xdfff; } + +[[nodiscard]] constexpr bool test_is_code_point(char32_t value) { return value <= 0x10ffff; } + +[[nodiscard]] constexpr bool test_is_scalar_value(char32_t value) { + return test_is_code_point(value) && !test_is_surrogate(value); +} + +inline constexpr char32_t test_replacement_character = U'\ufffd'; + +template +OutIt test_transcode() = delete; + +template + requires(std::output_iterator && std::same_as, char8_t>) +OutIt test_transcode(InIt first, InIt last, OutIt out_it) { + return std::copy(first, last, out_it); +} + +template + requires std::output_iterator +void test_encode(OutIt& out_it, char16_t value) { + if (value < 0x80) + *out_it++ = static_cast(value); + else if (value < 0x800) { + *out_it++ = static_cast(0b11000000 | (value >> 6)); + *out_it++ = static_cast(0b10000000 | (value & 0b00111111)); + } else { + *out_it++ = static_cast(0b11100000 | (value >> 12)); + *out_it++ = static_cast(0b10000000 | ((value) >> 6 & 0b00111111)); + *out_it++ = static_cast(0b10000000 | (value & 0b00111111)); + } +} + +template + requires std::output_iterator +void test_encode(OutIt& out_it, char32_t value) { + if ((value & 0xffff0000) == 0) + test_encode(out_it, static_cast(value)); + else { + *out_it++ = static_cast(0b11100000 | (value >> 18)); + *out_it++ = static_cast(0b10000000 | ((value) >> 12 & 0b00111111)); + *out_it++ = static_cast(0b10000000 | ((value) >> 6 & 0b00111111)); + *out_it++ = static_cast(0b10000000 | (value & 0b00111111)); + } +} + +template + requires(std::output_iterator && + (std::same_as, char16_t> +# ifndef TEST_HAS_NO_WIDE_CHARACTERS + || (std::same_as, wchar_t> && sizeof(wchar_t) == 2) +# endif + )) +OutIt test_transcode(InIt first, InIt last, OutIt out_it) { + while (first != last) { + char32_t value = *first++; + + if (test_is_low_surrogate(value)) [[unlikely]] { + test_encode(out_it, static_cast(test_replacement_character)); + continue; + } + + if (!test_is_high_surrogate(value)) { + test_encode(out_it, static_cast(value)); + continue; + } + + if (first == last || !test_is_low_surrogate(static_cast(*first))) [[unlikely]] { + test_encode(out_it, static_cast(test_replacement_character)); + continue; + } + + value -= 0xd800; + value <<= 10; + value += static_cast(*first++) - 0xdc00; + value += 0x10000; + + if (test_is_code_point(value)) [[likely]] + test_encode(out_it, value); + else + test_encode(out_it, static_cast(test_replacement_character)); + } + + return out_it; +} + +template + requires(std::output_iterator && + (std::same_as, char32_t> +# ifndef TEST_HAS_NO_WIDE_CHARACTERS + || (std::same_as, wchar_t> && sizeof(wchar_t) == 4) +# endif + )) +OutIt test_transcode(InIt first, InIt last, OutIt out_it) { + while (first != last) { + char32_t value = *first++; + if (test_is_code_point(value)) [[likely]] + test_encode(out_it, value); + else + test_encode(out_it, static_cast(test_replacement_character)); + } + return out_it; +} + +template +concept test_streamable = requires(std::stringstream& stream, T&& value) { stream << value; }; + +template +concept test_convertable_range = (!test_streamable && requires(R&& value) { + std::basic_string_view{std::begin(value), std::end(value)}; +}); + template -concept test_char_streamable = requires(T&& value) { std::stringstream{} << std::forward(value); }; -# endif +concept test_can_concat = test_streamable || test_convertable_range; + +template +std::ostream& test_concat(std::ostream& stream, T&& value) { + return stream << value; +} + +template +std::ostream& test_concat(std::ostream& stream, T&& value) { + auto b = std::begin(value); + auto e = std::end(value); + if (b != e) { + // When T is an array it's string-literal, remove the NUL terminator. + if constexpr (std::is_array_v>) { + --e; + } + test_transcode(b, e, std::ostream_iterator{stream}); + } + return stream; +} +# endif // TEST_HAS_NO_LOCALIZATION // If possible concatenates message for the assertion function, else returns a // default message. Not being able to stream is not considered an error. For @@ -37,12 +176,12 @@ concept test_char_streamable = requires(T&& value) { std::stringstream{} << std: template std::string test_concat_message([[maybe_unused]] Args&&... args) { # ifndef TEST_HAS_NO_LOCALIZATION - if constexpr ((test_char_streamable && ...)) { + if constexpr ((test_can_concat && ...)) { std::stringstream sstr; - ((sstr << std::forward(args)), ...); + ((test_concat(sstr, std::forward(args))), ...); return sstr.str(); } else -# endif +# endif // TEST_HAS_NO_LOCALIZATION return "Message discarded since it can't be streamed to std::cerr.\n"; } diff --git a/libcxx/test/support/msvc_stdlib_force_include.h b/libcxx/test/support/msvc_stdlib_force_include.h index b5ba33bd281c0009e49f207b60684a6101e8f1dd..5742bbf4f5572674c5d4edb444e1fc9fa2abfa26 100644 --- a/libcxx/test/support/msvc_stdlib_force_include.h +++ b/libcxx/test/support/msvc_stdlib_force_include.h @@ -100,6 +100,8 @@ const AssertionDialogAvoider assertion_dialog_avoider{}; # define TEST_STD_VER 14 #endif +#define TEST_SHORT_WCHAR + #define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST #ifdef __clang__ diff --git a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp index fba48d5984ea775a1267c91b709b387564e3ba72..fcb5865adf0d4e175a6a8a9310bb08aff896f1ff 100644 --- a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp +++ b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp @@ -26,7 +26,10 @@ struct clang::tidy::OptionEnumMapping Mapping[] = { {libcpp::header_exportable_declarations::FileType::Header, "Header"}, {libcpp::header_exportable_declarations::FileType::ModulePartition, "ModulePartition"}, - {libcpp::header_exportable_declarations::FileType::Module, "Module"}}; + {libcpp::header_exportable_declarations::FileType::Module, "Module"}, + {libcpp::header_exportable_declarations::FileType::CHeader, "CHeader"}, + {libcpp::header_exportable_declarations::FileType::CompatModulePartition, "CompatModulePartition"}, + {libcpp::header_exportable_declarations::FileType::CompatModule, "CompatModule"}}; return ArrayRef(Mapping); } }; @@ -39,6 +42,7 @@ header_exportable_declarations::header_exportable_declarations( file_type_(Options.get("FileType", header_exportable_declarations::FileType::Unknown)), extra_header_(Options.get("ExtraHeader", "")) { switch (file_type_) { + case header_exportable_declarations::FileType::CHeader: case header_exportable_declarations::FileType::Header: if (filename_.empty()) llvm::errs() << "No filename is provided.\n"; @@ -46,10 +50,12 @@ header_exportable_declarations::header_exportable_declarations( extra_header_ = "$^"; // Use a never matching regex to silence an error message. break; case header_exportable_declarations::FileType::ModulePartition: + case header_exportable_declarations::FileType::CompatModulePartition: if (filename_.empty()) llvm::errs() << "No filename is provided.\n"; [[fallthrough]]; case header_exportable_declarations::FileType::Module: + case header_exportable_declarations::FileType::CompatModule: if (!extra_header_.empty()) llvm::errs() << "Extra headers are not allowed for modules.\n"; if (Options.get("SkipDeclarations")) @@ -70,7 +76,7 @@ header_exportable_declarations::header_exportable_declarations( auto b = s.begin(); auto e = std::find(b, s.end(), ' '); while (b != e) { - decls_.emplace(b, e); + skip_decls_.emplace(b, e); if (e == s.end()) break; b = e + 1; @@ -82,9 +88,10 @@ header_exportable_declarations::header_exportable_declarations( for (auto decl : std::views::split(*list, ' ')) { std::string s; std::ranges::copy(decl, std::back_inserter(s)); // use range based constructor - decls_.emplace(std::move(s)); + skip_decls_.emplace(std::move(s)); } #endif // defined(__clang_major__) && __clang_major__ < 16 + decls_ = skip_decls_; list = Options.get("ExtraDeclarations"); // TODO(LLVM-17) Remove clang 15 work-around. @@ -94,7 +101,7 @@ header_exportable_declarations::header_exportable_declarations( auto b = s.begin(); auto e = std::find(b, s.end(), ' '); while (b != e) { - std::cout << "using " << std::string_view{b, e} << ";\n"; + std::cout << "using ::" << std::string_view{b, e} << ";\n"; if (e == s.end()) break; b = e + 1; @@ -104,10 +111,16 @@ header_exportable_declarations::header_exportable_declarations( #else // defined(__clang_major__) && __clang_major__ < 16 if (list) for (auto decl : std::views::split(*list, ' ')) - std::cout << "using " << std::string_view{decl.data(), decl.size()} << ";\n"; + std::cout << "using ::" << std::string_view{decl.data(), decl.size()} << ";\n"; #endif // defined(__clang_major__) && __clang_major__ < 16 } +header_exportable_declarations::~header_exportable_declarations() { + for (const auto& name : global_decls_) + if (!skip_decls_.contains("std::" + name) && decls_.contains("std::" + name)) + std::cout << "using ::" << name << ";\n"; +} + void header_exportable_declarations::registerMatchers(clang::ast_matchers::MatchFinder* finder) { // there are no public names in the Standard starting with an underscore, so // no need to check the strict rules. @@ -129,10 +142,22 @@ void header_exportable_declarations::registerMatchers(clang::ast_matchers::Match .bind("header_exportable_declarations"), this); break; + case FileType::CHeader: + // For C headers of the std.compat two matchers are used + // - The cheader matcher; in libc++ these are never split in multiple + // headers so limiting the declarations to that header works. + // - The header.h; where the declarations of this header are provided + // is not specified and depends on the libc used. Therefore it is not + // possible to restrict the location in a portable way. + finder->addMatcher(namedDecl().bind("cheader_exportable_declarations"), this); + + [[fallthrough]]; case FileType::ModulePartition: + case FileType::CompatModulePartition: finder->addMatcher(namedDecl(isExpansionInFileMatching(filename_)).bind("header_exportable_declarations"), this); break; case FileType::Module: + case FileType::CompatModule: finder->addMatcher(namedDecl().bind("header_exportable_declarations"), this); break; case header_exportable_declarations::FileType::Unknown: @@ -156,6 +181,9 @@ void header_exportable_declarations::registerMatchers(clang::ast_matchers::Match /// * cstddef has bitwise operators for the type \c byte /// * exception has equality operators for the type \c exception_ptr /// * initializer_list has the functions \c begin and \c end +/// +/// \warning In some cases the returned name can be an empty string. +/// The cause has not been investigated. static std::string get_qualified_name(const clang::NamedDecl& decl) { std::string result = decl.getQualifiedNameAsString(); @@ -166,10 +194,6 @@ static std::string get_qualified_name(const clang::NamedDecl& decl) { } static bool is_viable_declaration(const clang::NamedDecl* decl) { - // Declarations nested in records are automatically exported with the record itself. - if (!decl->getDeclContext()->isNamespace()) - return false; - // Declarations that are a subobject of a friend Declaration are automatically exported with the record itself. if (decl->getFriendObjectKind() != clang::Decl::FOK_None) return false; @@ -199,13 +223,21 @@ static bool is_viable_declaration(const clang::NamedDecl* decl) { /// Returns the name is a reserved name. /// /// Detected reserved names are names starting with __ or _[A-Z]. -/// These names can be in the namespace std or any namespace inside std. For -/// example std::ranges contains reserved names to implement the Niebloids. +/// These names can be in the global namespace, std namespace or any namespace +/// inside std. For example, std::ranges contains reserved names to implement +/// the Niebloids. /// -/// This test misses 2 candidates which are not used in libc++ +/// This test misses candidates which are not used in libc++ /// * any identifier with two underscores not at the start -/// * a name with a leading underscore in the global namespace -bool is_reserved_name(const std::string& name) { +bool is_reserved_name(std::string_view name) { + if (name.starts_with("_")) { + // This is a public name declared in cstdlib. + if (name == "_Exit") + return false; + + return name.size() > 1 && (name[1] == '_' || std::isupper(name[1])); + } + std::size_t pos = name.find("::_"); if (pos == std::string::npos) return false; @@ -213,27 +245,73 @@ bool is_reserved_name(const std::string& name) { if (pos + 3 > name.size()) return false; + // This is a public name declared in cstdlib. + if (name == "std::_Exit") + return false; + return name[pos + 3] == '_' || std::isupper(name[pos + 3]); } +/// Some declarations in the global namespace are exported from the std module. +static bool is_global_name_exported_by_std_module(std::string_view name) { + static const std::set valid{ + "operator delete", "operator delete[]", "operator new", "operator new[]"}; + return valid.contains(name); +} + +static bool is_valid_declaration_context( + const clang::NamedDecl& decl, std::string_view name, header_exportable_declarations::FileType file_type) { + if (decl.getDeclContext()->isNamespace()) + return true; + + if (is_global_name_exported_by_std_module(name)) + return true; + + return file_type != header_exportable_declarations::FileType::Header; +} + +static bool is_module(header_exportable_declarations::FileType file_type) { + switch (file_type) { + case header_exportable_declarations::FileType::Module: + case header_exportable_declarations::FileType::ModulePartition: + case header_exportable_declarations::FileType::CompatModule: + case header_exportable_declarations::FileType::CompatModulePartition: + return true; + + case header_exportable_declarations::FileType::Header: + case header_exportable_declarations::FileType::CHeader: + return false; + + case header_exportable_declarations::FileType::Unknown: + llvm::errs() << "This should be unreachable.\n"; + break; + } +} + void header_exportable_declarations::check(const clang::ast_matchers::MatchFinder::MatchResult& result) { if (const auto* decl = result.Nodes.getNodeAs("header_exportable_declarations"); decl != nullptr) { if (!is_viable_declaration(decl)) return; std::string name = get_qualified_name(*decl); + if (name.empty()) + return; + if (is_reserved_name(name)) return; // For modules only take the declarations exported. - if (file_type_ == FileType::ModulePartition || file_type_ == FileType::Module) + if (is_module(file_type_)) if (decl->getModuleOwnershipKind() != clang::Decl::ModuleOwnershipKind::VisibleWhenImported) return; + if (!is_valid_declaration_context(*decl, name, file_type_)) + return; + if (decls_.contains(name)) { // For modules avoid exporting the same named declaration twice. For // header files this is common and valid. - if (file_type_ == FileType::ModulePartition) + if (file_type_ == FileType::ModulePartition || file_type_ == FileType::CompatModulePartition) // After the warning the script continues. // The test will fail since modules have duplicated entries and headers not. llvm::errs() << "Duplicated export of '" << name << "'.\n"; @@ -241,8 +319,30 @@ void header_exportable_declarations::check(const clang::ast_matchers::MatchFinde return; } - std::cout << "using " << std::string{name} << ";\n"; + // For named declarations in std this is valid + // using std::foo; + // for named declarations it is invalid to use + // using bar; + // Since fully qualifying named declarations in the std namespace is valid + // using fully qualified names unconditionally. + std::cout << "using ::" << std::string{name} << ";\n"; decls_.insert(name); + } else if (const auto* decl = result.Nodes.getNodeAs("cheader_exportable_declarations"); + decl != nullptr) { + if (decl->getDeclContext()->isNamespace()) + return; + + if (!is_viable_declaration(decl)) + return; + + std::string name = get_qualified_name(*decl); + if (is_reserved_name(name)) + return; + + if (global_decls_.contains(name)) + return; + + global_decls_.insert(name); } } diff --git a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.hpp b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.hpp index 119bcd3f6f2fe5a72fc5d9206200b13ce710bdcc..5d9e0f3ef9c30e2ccc6e5ef17dad4b90da0b7281 100644 --- a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.hpp +++ b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.hpp @@ -18,15 +18,47 @@ namespace libcpp { class header_exportable_declarations : public clang::tidy::ClangTidyCheck { public: explicit header_exportable_declarations(llvm::StringRef, clang::tidy::ClangTidyContext*); + ~header_exportable_declarations(); void registerMatchers(clang::ast_matchers::MatchFinder*) override; void check(const clang::ast_matchers::MatchFinder::MatchResult&) override; - enum class FileType { Header, ModulePartition, Module, Unknown }; + enum class FileType { + // std module specific + Header, + CompatModulePartition, + Module, + // std.compat module specific + CHeader, + ModulePartition, + CompatModule, + // invalid value + Unknown + }; private: llvm::StringRef filename_; FileType file_type_; llvm::StringRef extra_header_; std::set decls_; + std::set global_decls_; + + // The named declarations in .h C headers are "tricky". On POSIX + // systems these headers contain POSIX specific functions that do not + // use a reserved name. For example, fmemopen is provided by stdio.h. + // We filter the names that should be provided by the headers as follows: + // - record all named declarations the global namespace + // - wait until the header is completely processed + // - every named declaration in the global namespace that has a matching + // "export" in the std namespace is exported. + // + // The only place where we can do the above while ensuring that all + // the declarations in the header have been seen is in the clang tidy + // plugin's destructor. + // + // It is possible to skip some declarations in the std namespace, + // these are added to decls_ before processing. To differentiate + // between a skipped declaration and a real declaration the skipped + // declarations are recorded in an extra variable. + std::set skip_decls_; }; } // namespace libcpp diff --git a/libcxx/utils/CMakeLists.txt b/libcxx/utils/CMakeLists.txt index 7e597f632b6c46d9b6d1b92f5acec6ed06b060d4..19bb9851c8674cf890b0dd66c1c4e0dec4eec521 100644 --- a/libcxx/utils/CMakeLists.txt +++ b/libcxx/utils/CMakeLists.txt @@ -7,9 +7,19 @@ add_custom_target(libcxx-generate-std-clang-module-header COMMENT "Generate the <__std_clang_module> header") add_custom_target(libcxx-generate-std-cppm-in-file - COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/generate_std_cppm_in.py" + COMMAND + "${Python3_EXECUTABLE}" + "${LIBCXX_SOURCE_DIR}/utils/generate_libcxx_cppm_in.py" + "std" COMMENT "Generate the std.cppm.in file") +add_custom_target(libcxx-generate-std-compat-cppm-in-file + COMMAND + "${Python3_EXECUTABLE}" + "${LIBCXX_SOURCE_DIR}/utils/generate_libcxx_cppm_in.py" + "std.compat" + COMMENT "Generate the std.compat.cppm.in file") + add_custom_target(libcxx-generate-extended-grapheme-cluster-tables COMMAND "${Python3_EXECUTABLE}" @@ -48,6 +58,7 @@ add_custom_target(libcxx-generate-files DEPENDS libcxx-generate-feature-test-macros libcxx-generate-std-clang-module-header libcxx-generate-std-cppm-in-file + libcxx-generate-std-compat-cppm-in-file libcxx-generate-extended-grapheme-cluster-tables libcxx-generate-extended-grapheme-cluster-tests libcxx-generate-escaped-output-table diff --git a/libcxx/utils/generate_std_cppm_in.py b/libcxx/utils/generate_libcxx_cppm_in.py similarity index 51% rename from libcxx/utils/generate_std_cppm_in.py rename to libcxx/utils/generate_libcxx_cppm_in.py index 242134773e6891cdb03ef9aacda76ce552c2688d..f957406778d39250167b7c977f52621e3dff9135 100644 --- a/libcxx/utils/generate_std_cppm_in.py +++ b/libcxx/utils/generate_libcxx_cppm_in.py @@ -7,20 +7,22 @@ # ===----------------------------------------------------------------------===## import os.path +import sys from libcxx.header_information import module_headers from libcxx.header_information import header_restrictions from libcxx.header_information import headers_not_available -libcxx_module_directory = os.path.join( - os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "modules" -) -with open( - os.path.join(libcxx_module_directory, "std.cppm.in"), "w" -) as std_module_cpp_in: - std_module_cpp_in.write( - """\ +def write_file(module): + libcxx_module_directory = os.path.join( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "modules" + ) + with open( + os.path.join(libcxx_module_directory, f"{module}.cppm.in"), "w" + ) as module_cpp_in: + module_cpp_in.write( + """\ // -*- C++ -*- //===----------------------------------------------------------------------===// // @@ -31,7 +33,7 @@ with open( //===----------------------------------------------------------------------===// // WARNING, this entire header is generated by -// utils/generate_std_cppm_in.py +// utils/generate_libcxx_cppm_in.py // DO NOT MODIFY! module; @@ -41,33 +43,46 @@ module; // The headers of Table 24: C++ library headers [tab:headers.cpp] // and the headers of Table 25: C++ headers for C library facilities [tab:headers.cpp.c] """ - ) - for header in module_headers: - if header in header_restrictions: - std_module_cpp_in.write( - f"""\ + ) + for header in module_headers: + if header in header_restrictions: + module_cpp_in.write( + f"""\ #if {header_restrictions[header]} # include <{header}> #endif """ - ) - else: - std_module_cpp_in.write(f"#include <{header}>\n") + ) + else: + module_cpp_in.write(f"#include <{header}>\n") - std_module_cpp_in.write("\n// *** Headers not yet available ***\n") - for header in sorted(headers_not_available): - std_module_cpp_in.write( - f"""\ + module_cpp_in.write("\n// *** Headers not yet available ***\n") + for header in sorted(headers_not_available): + module_cpp_in.write( + f"""\ #if __has_include(<{header}>) -# error "update the header information for <{header}> in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include(<{header}>) +# error "please update the header information for <{header}> in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include(<{header}>) """ - ) + ) - std_module_cpp_in.write( - """ -export module std; + module_cpp_in.write( + f""" +export module {module}; @LIBCXX_MODULE_STD_INCLUDE_SOURCES@ +{'@LIBCXX_MODULE_STD_COMPAT_INCLUDE_SOURCES@' if module == 'std.compat' else ''}""" + ) + + +if __name__ == "__main__": + if len(sys.argv) != 2 or (sys.argv[1] != "std" and sys.argv[1] != "std.compat"): + sys.stderr.write( + f"""\ +Usage: +{os.path.basename(__file__)} (std|std.compat) """ - ) + ) + sys.exit(1) + + write_file(sys.argv[1]) diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py index ccabb48833f10def1d61962af87d2bd7e618f0a0..5e854917e6ef45f5746f825a57e26189a36c8f39 100644 --- a/libcxx/utils/libcxx/test/features.py +++ b/libcxx/utils/libcxx/test/features.py @@ -198,6 +198,18 @@ DEFAULT_FEATURES = [ """, ), ), + # Tests that require 64-bit architecture + Feature( + name="32-bit-pointer", + when=lambda cfg: sourceBuilds( + cfg, + """ + int main(int, char**) { + static_assert(sizeof(void *) == 4); + } + """, + ), + ), # TODO: Remove this feature once compiler-rt includes __atomic_is_lockfree() # on all supported platforms. Feature( @@ -567,16 +579,4 @@ DEFAULT_FEATURES += [ cfg.available_features, ), ), - # Tests that require 64-bit architecture - Feature( - name="32-bit-pointer", - when=lambda cfg: sourceBuilds( - cfg, - """ - int main(int, char**) { - static_assert(sizeof(void *) == 4); - } - """, - ), - ), ] diff --git a/libcxx/utils/libcxx/test/modules.py b/libcxx/utils/libcxx/test/modules.py new file mode 100644 index 0000000000000000000000000000000000000000..deaac450381c38761fce4b752a33e192db0d7a4d --- /dev/null +++ b/libcxx/utils/libcxx/test/modules.py @@ -0,0 +1,302 @@ +# ===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------===## + +from libcxx.header_information import module_headers +from libcxx.header_information import header_restrictions +from dataclasses import dataclass + +### SkipDeclarations + +# Ignore several declarations found in the includes. +# +# Part of these items are bugs other are not yet implemented features. +SkipDeclarations = dict() + +# See comment in the header. +SkipDeclarations["cuchar"] = ["std::mbstate_t", "std::size_t"] + +# Not in the synopsis. +SkipDeclarations["cwchar"] = ["std::FILE"] + +# The operators are added for private types like __iom_t10. +SkipDeclarations["iomanip"] = ["std::operator<<", "std::operator>>"] + +SkipDeclarations["iosfwd"] = ["std::ios_base", "std::vector"] + +# This header also provides declarations in the namespace that might be +# an error. +SkipDeclarations["filesystem"] = [ + "std::filesystem::operator==", + "std::filesystem::operator!=", +] + +# This is a specialization for a private type +SkipDeclarations["iterator"] = ["std::pointer_traits"] + +# TODO MODULES +# This definition is declared in string and defined in istream +# This declaration should be part of string +SkipDeclarations["istream"] = ["std::getline"] + +# P1614 (at many places) and LWG3519 too. +SkipDeclarations["random"] = [ + "std::operator!=", + # LWG3519 makes these hidden friends. + # Note the older versions had the requirement of these operations but not in + # the synopsis. + "std::operator<<", + "std::operator>>", + "std::operator==", +] + +# Declared in the forward header since std::string uses std::allocator +SkipDeclarations["string"] = ["std::allocator"] +# TODO MODULES remove zombie names +# https://libcxx.llvm.org/Status/Cxx20.html#note-p0619 +SkipDeclarations["memory"] = [ + "std::return_temporary_buffer", + "std::get_temporary_buffer", +] + +# TODO MODULES this should be part of ios instead +SkipDeclarations["streambuf"] = ["std::basic_ios"] + +# include/__type_traits/is_swappable.h +SkipDeclarations["type_traits"] = [ + "std::swap", + # TODO MODULES gotten through __functional/unwrap_ref.h + "std::reference_wrapper", +] + +### ExtraDeclarations + +# Add declarations in headers. +# +# Some headers have their defines in a different header, which may have +# additional declarations. +ExtraDeclarations = dict() +# This declaration is in the ostream header. +ExtraDeclarations["system_error"] = ["std::operator<<"] + +### ExtraHeader + +# Adds extra headers file to scan +# +# Some C++ headers in libc++ are stored in multiple physical files. There is a +# pattern to find these files. However there are some exceptions these are +# listed here. +ExtraHeader = dict() +# locale has a file and not a subdirectory +ExtraHeader["locale"] = "v1/__locale$" +ExtraHeader["thread"] = "v1/__threading_support$" +ExtraHeader["ranges"] = "v1/__fwd/subrange.h$" + +# The extra header is needed since two headers are required to provide the +# same definition. +ExtraHeader["functional"] = "v1/__compare/compare_three_way.h$" + + +# newline needs to be escaped for the module partition output. +nl = "\\\\n" + + +@dataclass +class module_test_generator: + tmp_prefix: str + module_path: str + clang_tidy: str + clang_tidy_plugin: str + compiler: str + compiler_flags: str + + def write_lit_configuration(self): + print( + f"""\ +// UNSUPPORTED: c++03, c++11, c++14, c++17 +// UNSUPPORTED: libcpp-has-no-std-modules +// UNSUPPORTED: clang-modules-build + +// REQUIRES: has-clang-tidy + +// The GCC compiler flags are not always compatible with clang-tidy. +// UNSUPPORTED: gcc + +// RUN: echo -n > {self.tmp_prefix}.all_partitions +""" + ) + + def process_module_partition(self, header, is_c_header): + # Some headers cannot be included when a libc++ feature is disabled. + # In that case include the header conditionally. The header __config + # ensures the libc++ feature macros are available. + if header in header_restrictions: + include = ( + f"#include <__config>{nl}" + f"#if {header_restrictions[header]}{nl}" + f"# include <{header}>{nl}" + f"#endif{nl}" + ) + elif header == "chrono": + # When localization is disabled the header string is not included. + # When string is included chrono's operator""s is a named declaration + # using std::chrono_literals::operator""s; + # else it is a named declaration + # using std::operator""s; + # TODO MODULES investigate why + include = f"#include {nl}#include {nl}" + else: + include = f"#include <{header}>{nl}" + + module_files = f'#include \\"{self.module_path}/std/{header}.inc\\"{nl}' + if is_c_header: + module_files += ( + f'#include \\"{self.module_path}/std.compat/{header}.inc\\"{nl}' + ) + + # Generate a module partition for the header module includes. This + # makes it possible to verify that all headers export all their + # named declarations. + print( + '// RUN: echo -e "' + f"module;{nl}" + f"{include}{nl}" + f"{nl}" + f"// Use __libcpp_module_
to ensure that modules{nl}" + f"// are not named as keywords or reserved names.{nl}" + f"export module std:__libcpp_module_{header};{nl}" + f"{module_files}" + f'" > {self.tmp_prefix}.{header}.cppm' + ) + + # Extract the information of the module partition using lang-tidy + print( + f"// RUN: {self.clang_tidy} {self.tmp_prefix}.{header}.cppm " + " --checks='-*,libcpp-header-exportable-declarations' " + " -config='{CheckOptions: [ " + " {" + " key: libcpp-header-exportable-declarations.Filename, " + f" value: {header}.inc" + " }, {" + " key: libcpp-header-exportable-declarations.FileType, " + f" value: {'CompatModulePartition' if is_c_header else 'ModulePartition'}" + " }, " + " ]}' " + f"--load={self.clang_tidy_plugin} " + f"-- {self.compiler_flags} " + f"| sort > {self.tmp_prefix}.{header}.module" + ) + print( + f"// RUN: cat {self.tmp_prefix}.{header}.module >> {self.tmp_prefix}.all_partitions" + ) + + return include + + def process_header(self, header, include, is_c_header): + # Dump the information as found in the module by using the header file(s). + skip_declarations = " ".join(SkipDeclarations.get(header, [])) + if skip_declarations: + skip_declarations = ( + "{" + " key: libcpp-header-exportable-declarations.SkipDeclarations, " + f' value: "{skip_declarations}" ' + "}, " + ) + + extra_declarations = " ".join(ExtraDeclarations.get(header, [])) + if extra_declarations: + extra_declarations = ( + "{" + " key: libcpp-header-exportable-declarations.ExtraDeclarations, " + f' value: "{extra_declarations}" ' + "}, " + ) + + extra_header = ExtraHeader.get(header, "") + if extra_header: + extra_header = ( + "{" + " key: libcpp-header-exportable-declarations.ExtraHeader, " + f' value: "{extra_header}" ' + "}, " + ) + + # Clang-tidy needs a file input + print(f'// RUN: echo -e "' f"{include}" f'" > {self.tmp_prefix}.{header}.cpp') + print( + f"// RUN: {self.clang_tidy} {self.tmp_prefix}.{header}.cpp " + " --checks='-*,libcpp-header-exportable-declarations' " + " -config='{CheckOptions: [ " + " {" + " key: libcpp-header-exportable-declarations.Filename, " + f" value: {header}" + " }, {" + " key: libcpp-header-exportable-declarations.FileType, " + f" value: {'CHeader' if is_c_header else 'Header'}" + " }, " + f" {skip_declarations} {extra_declarations} {extra_header}, " + " ]}' " + f"--load={self.clang_tidy_plugin} " + f"-- {self.compiler_flags} " + f"| sort > {self.tmp_prefix}.{header}.include" + ) + print( + f"// RUN: diff -u {self.tmp_prefix}.{header}.module {self.tmp_prefix}.{header}.include" + ) + + def process_module(self, module): + # Merge the data of the parts + print( + f"// RUN: sort -u -o {self.tmp_prefix}.all_partitions {self.tmp_prefix}.all_partitions" + ) + + # Dump the information as found in top-level module. + print( + f"// RUN: {self.clang_tidy} {self.module_path}/{module}.cppm " + " --checks='-*,libcpp-header-exportable-declarations' " + " -config='{CheckOptions: [ " + " {" + " key: libcpp-header-exportable-declarations.Header, " + f" value: {module}.cppm" + " }, {" + " key: libcpp-header-exportable-declarations.FileType, " + " value: Module" + " }, " + " ]}' " + f"--load={self.clang_tidy_plugin} " + f"-- {self.compiler_flags} " + f"| sort > {self.tmp_prefix}.module" + ) + + # Compare the sum of the parts with the top-level module. + print( + f"// RUN: diff -u {self.tmp_prefix}.all_partitions {self.tmp_prefix}.module" + ) + + # Basic smoke test. Import a module and try to compile when using all + # exported names. This validates the clang-tidy script does not + # accidentally add named declarations to the list that are not available. + def test_module(self, module): + print( + f"""\ +// RUN: echo 'import {module};' > {self.tmp_prefix}.compile.pass.cpp +// RUN: cat {self.tmp_prefix}.all_partitions >> {self.tmp_prefix}.compile.pass.cpp +// RUN: {self.compiler} {self.compiler_flags} -fsyntax-only {self.tmp_prefix}.compile.pass.cpp +""" + ) + + def write_test(self, module, c_headers=[]): + self.write_lit_configuration() + + # Validate all module parts. + for header in module_headers: + is_c_header = header in c_headers + include = self.process_module_partition(header, is_c_header) + self.process_header(header, include, is_c_header) + + self.process_module(module) + self.test_module(module) diff --git a/libcxxabi/src/cxa_personality.cpp b/libcxxabi/src/cxa_personality.cpp index 4570d0b5beb2e2e95fbcfa5e9166156e82bc4643..4b6c4edbc2669851a0bd1936399b1d225c39e160 100644 --- a/libcxxabi/src/cxa_personality.cpp +++ b/libcxxabi/src/cxa_personality.cpp @@ -660,10 +660,9 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, // dwarf emission // Parse LSDA header. uint8_t lpStartEncoding = *lsda++; - const uint8_t* lpStart = - (const uint8_t*)readEncodedPointer(&lsda, lpStartEncoding, base); - if (lpStart == 0) - lpStart = (const uint8_t*)funcStart; + const uint8_t* lpStart = lpStartEncoding == DW_EH_PE_omit + ? (const uint8_t*)funcStart + : (const uint8_t*)readEncodedPointer(&lsda, lpStartEncoding, base); uint8_t ttypeEncoding = *lsda++; if (ttypeEncoding != DW_EH_PE_omit) { diff --git a/libcxxabi/test/native/x86_64/lpstart-zero.pass.sh.s b/libcxxabi/test/native/x86_64/lpstart-zero.pass.sh.s new file mode 100644 index 0000000000000000000000000000000000000000..ea9dd0104c086bddf780fa59113e8363be4c0596 --- /dev/null +++ b/libcxxabi/test/native/x86_64/lpstart-zero.pass.sh.s @@ -0,0 +1,109 @@ +# RUN: %{cxx} %{flags} %s %{link_flags} -no-pie -o %t.exe +# RUN: %t.exe + +# REQUIRES: linux && target={{x86_64-.+}} +# UNSUPPORTED: no-exceptions + +## Check that libc++abi works correctly when LPStart address is explicitly set +## to zero. + +## This file is generated from the following C++ source code. +## +## ``` +## int main() { +## try { +## throw 42; +## } catch (...) { +## return 0; +## } +## return 1; +## } +## ``` +## The exception table is modified to use udata4 encoding for LPStart and +## sdata4 encoding for call sites. + + .text + .globl main # -- Begin function main + .p2align 4, 0x90 + .type main,@function +main: # @main +.Lfunc_begin0: + .cfi_startproc + .globl __gxx_personality_v0 + .cfi_personality 3, __gxx_personality_v0 + .cfi_lsda 27, .Lexception0 +# %bb.0: # %entry + pushq %rbp + .cfi_def_cfa_offset 16 + .cfi_offset %rbp, -16 + movq %rsp, %rbp + .cfi_def_cfa_register %rbp + subq $32, %rsp + movl $0, -4(%rbp) + movl $4, %edi + callq __cxa_allocate_exception@PLT + movq %rax, %rdi + movl $42, (%rdi) +.Ltmp0: + movq _ZTIi@GOTPCREL(%rip), %rsi + xorl %eax, %eax + movl %eax, %edx + callq __cxa_throw@PLT +.Ltmp1: + jmp .LBB0_4 +.LBB0_1: # %lpad +.Ltmp2: + movq %rax, %rcx + movl %edx, %eax + movq %rcx, -16(%rbp) + movl %eax, -20(%rbp) +# %bb.2: # %catch + movq -16(%rbp), %rdi + callq __cxa_begin_catch@PLT + movl $0, -4(%rbp) + callq __cxa_end_catch@PLT +# %bb.3: # %return + movl -4(%rbp), %eax + addq $32, %rsp + popq %rbp + .cfi_def_cfa %rsp, 8 + retq +.LBB0_4: # %unreachable +.Lfunc_end0: + .size main, .Lfunc_end0-main + .cfi_endproc + + .section .gcc_except_table,"a",@progbits + .p2align 2, 0x0 +GCC_except_table0: +.Lexception0: + .byte 3 # @LPStart Encoding = udata4 + .long 0 + .byte 155 # @TType Encoding = indirect pcrel sdata4 + .uleb128 .Lttbase0-.Lttbaseref0 +.Lttbaseref0: + .byte 11 # Call site Encoding = udata4 + .uleb128 .Lcst_end0-.Lcst_begin0 +.Lcst_begin0: + .long .Lfunc_begin0-.Lfunc_begin0 # >> Call Site 1 << + .long .Ltmp0-.Lfunc_begin0 # Call between .Lfunc_begin0 and .Ltmp0 + .long 0 # has no landing pad + .byte 0 # On action: cleanup + .long .Ltmp0-.Lfunc_begin0 # >> Call Site 2 << + .long .Ltmp1-.Ltmp0 # Call between .Ltmp0 and .Ltmp1 + .long .Ltmp2 + .byte 1 # On action: 1 + .long .Ltmp1-.Lfunc_begin0 # >> Call Site 3 << + .long .Lfunc_end0-.Ltmp1 # Call between .Ltmp1 and .Lfunc_end0 + .long 0 # has no landing pad + .byte 0 # On action: cleanup +.Lcst_end0: + .byte 1 # >> Action Record 1 << + # Catch TypeInfo 1 + .byte 0 # No further actions + .p2align 2, 0x0 + # >> Catch TypeInfos << + .long 0 # TypeInfo 1 +.Lttbase0: + .p2align 2, 0x0 + # -- End function diff --git a/lld/COFF/PDB.cpp b/lld/COFF/PDB.cpp index f77ff0d4eab804b7c03309644255de74b649a887..8b81a71e8fb6f979396d643ff4e7c7f0db071166 100644 --- a/lld/COFF/PDB.cpp +++ b/lld/COFF/PDB.cpp @@ -268,7 +268,7 @@ void PDBLinker::pdbMakeAbsolute(SmallVectorImpl &fileName) { // decide that it's a unix path if we're fairly certain. Specifically, if // it starts with a forward slash. SmallString<128> absoluteFileName = ctx.config.pdbSourcePath; - sys::path::Style guessedStyle = absoluteFileName.startswith("/") + sys::path::Style guessedStyle = absoluteFileName.starts_with("/") ? sys::path::Style::posix : sys::path::Style::windows; sys::path::append(absoluteFileName, guessedStyle, fileName); diff --git a/lld/ELF/DWARF.h b/lld/ELF/DWARF.h index e1688fef9c9e47c13037c963dbb9b55a5cc8aa64..1b9a3e3f77943b19c94cd84b19801a9772814c46 100644 --- a/lld/ELF/DWARF.h +++ b/lld/ELF/DWARF.h @@ -13,6 +13,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/Object/ELF.h" +#include namespace lld::elf { diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp index 06a3d565deb765ef763c76ee4256b0b115a87a2d..cc2c5916e05c22c425972e5b96ce53ff03b08c0e 100644 --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -30,6 +30,7 @@ #include "llvm/Support/RISCVAttributeParser.h" #include "llvm/Support/TarWriter.h" #include "llvm/Support/raw_ostream.h" +#include using namespace llvm; using namespace llvm::ELF; diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp index 6e972cf084cdc62327aa30314a46882b97c142b0..81468a20dfb54dfa0fe928917e65d22c65b2c9f6 100644 --- a/lld/ELF/InputSection.cpp +++ b/lld/ELF/InputSection.cpp @@ -22,6 +22,7 @@ #include "llvm/Support/xxhash.h" #include #include +#include #include using namespace llvm; @@ -909,16 +910,8 @@ void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef rels) { for (size_t i = 0, relsSize = rels.size(); i != relsSize; ++i) { const RelTy &rel = rels[i]; - RelType type = rel.getType(config->isMips64EL); - - // GCC 8.0 or earlier have a bug that they emit R_386_GOTPC relocations - // against _GLOBAL_OFFSET_TABLE_ for .debug_info. The bug has been fixed - // in 2017 (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82630), but we - // need to keep this bug-compatible code for a while. - if (emachine == EM_386 && type == R_386_GOTPC) - continue; - - uint64_t offset = rel.r_offset; + const RelType type = rel.getType(config->isMips64EL); + const uint64_t offset = rel.r_offset; uint8_t *bufLoc = buf + offset; int64_t addend = getAddend(rel); if (!RelTy::IsRela) @@ -998,25 +991,25 @@ void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef rels) { if (config->relocatable && (RelTy::IsRela || sym.type != STT_SECTION)) continue; - if (expr == R_SIZE) { - target.relocateNoSym(bufLoc, type, - SignExtend64(sym.getSize() + addend)); - continue; - } - // R_ABS/R_DTPREL and some other relocations can be used from non-SHF_ALLOC // sections. - if (expr == R_ABS || expr == R_DTPREL || expr == R_GOTPLTREL || + if (LLVM_LIKELY(expr == R_ABS) || expr == R_DTPREL || expr == R_GOTPLTREL || expr == R_RISCV_ADD) { target.relocateNoSym(bufLoc, type, SignExtend64(sym.getVA(addend))); continue; } + if (expr == R_SIZE) { + target.relocateNoSym(bufLoc, type, + SignExtend64(sym.getSize() + addend)); + continue; + } + std::string msg = getLocation(offset) + ": has non-ABS relocation " + toString(type) + " against symbol '" + toString(sym) + "'"; - if (expr != R_PC) { - error(msg); + if (expr != R_PC && !(emachine == EM_386 && type == R_386_GOTPC)) { + errorOrWarn(msg); return; } @@ -1028,11 +1021,11 @@ void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef rels) { // address 0. For bug-compatibility, we accept them with warnings. We // know Steel Bank Common Lisp as of 2018 have this bug. // - // RELA -r stopped earlier and does not get the warning. Suppress the - // warning for REL -r as well - // (https://github.com/ClangBuiltLinux/linux/issues/1937). - if (RelTy::IsRela || !config->relocatable) - warn(msg); + // GCC 8.0 or earlier have a bug that they emit R_386_GOTPC relocations + // against _GLOBAL_OFFSET_TABLE_ for .debug_info. The bug has been fixed in + // 2017 (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82630), but we need to + // keep this bug-compatible code for a while. + warn(msg); target.relocateNoSym( bufLoc, type, SignExtend64(sym.getVA(addend - offset - outSecOff))); diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp index 31ed24149e78ddf6d0cd96241eb84b0da564c920..7d0cdce9de7dcb792d19498e33a1771692badd69 100644 --- a/lld/MachO/InputFiles.cpp +++ b/lld/MachO/InputFiles.cpp @@ -1535,7 +1535,7 @@ std::string ObjFile::sourceFile() const { StringRef sep = sys::path::get_separator(); // We don't use `path::append` here because we want an empty `dir` to result // in an absolute path. `append` would give us a relative path for that case. - if (!dir.endswith(sep)) + if (!dir.ends_with(sep)) dir += sep; return (dir + unitName).str(); } diff --git a/lld/test/ELF/debug-dead-reloc-32.s b/lld/test/ELF/debug-dead-reloc-32.s index b2708a744f2887681b61f05d728a2ffe2b1e58be..99335b44f51cedff9166e4be4497bd469558c781 100644 --- a/lld/test/ELF/debug-dead-reloc-32.s +++ b/lld/test/ELF/debug-dead-reloc-32.s @@ -14,6 +14,17 @@ # CHECK-NEXT: Contents of section .debug_addr: # CHECK-NEXT: 0000 00000000 +## -z dead-reloc-in-nonalloc= can override the tombstone value. +# RUN: ld.lld -z dead-reloc-in-nonalloc=.debug_loc=42 -z dead-reloc-in-nonalloc=.debug_addr=0xfffffffffffffffe %t.o -o %t1 +# RUN: llvm-objdump -s %t1 | FileCheck %s --check-prefix=OVERRIDE + +# OVERRIDE: Contents of section .debug_loc: +# OVERRIDE-NEXT: 0000 2a000000 *... +# OVERRIDE-NEXT: Contents of section .debug_ranges: +# OVERRIDE-NEXT: 0000 01000000 .... +# OVERRIDE-NEXT: Contents of section .debug_addr: +# OVERRIDE-NEXT: 0000 feffffff .... + .section .text.1,"axe" .byte 0 diff --git a/lld/test/ELF/debug-dead-reloc.s b/lld/test/ELF/debug-dead-reloc.s index fcf53205079ed6a58635f60d9f00057b80e61b2c..cfa41e00eab06d25fbe07f662c0ca884491900aa 100644 --- a/lld/test/ELF/debug-dead-reloc.s +++ b/lld/test/ELF/debug-dead-reloc.s @@ -1,4 +1,4 @@ -# REQUIRES: x86 +# REQUIRES: aarch64, x86 ## Test we resolve symbolic relocations in .debug_* sections to a tombstone ## value if the referenced symbol is discarded (--gc-sections, non-prevailing ## section group, SHF_EXCLUDE, /DISCARD/, etc). @@ -9,13 +9,18 @@ # RUN: llvm-objdump -s %t | FileCheck %s # RUN: llvm-readobj -r %t | FileCheck %s --check-prefix=REL +# RUN: echo '.globl _start; _start: bl group' | llvm-mc -filetype=obj -triple=aarch64 - -o %t.a64.o +# RUN: llvm-mc -filetype=obj -triple=aarch64 %s -o %t1.a64.o +# RUN: ld.lld --emit-relocs --gc-sections %t.a64.o %t1.a64.o %t1.a64.o -o %t.a64 +# RUN: llvm-objdump -s %t.a64 | FileCheck %s + # CHECK: Contents of section .debug_loc: # CHECK-NEXT: 0000 01000000 00000000 01000000 00000000 # CHECK: Contents of section .debug_ranges: # CHECK-NEXT: 0000 01000000 00000000 01000000 00000000 # CHECK: Contents of section .debug_addr: -# CHECK-NEXT: 0000 {{.*}}000 00000000 {{.*}}000 00000000 -# CHECK-NEXT: 0010 00000000 00000000 {{.*}}000 00000000 +# CHECK-NEXT: 0000 {{.*}}00 00000000 {{.*}}00 00000000 +# CHECK-NEXT: 0010 00000000 00000000 {{.*}}00 00000000 # CHECK: Contents of section .debug_foo: # CHECK-NEXT: 0000 00000000 00000000 08000000 00000000 # CHECK-NEXT: 0010 00000000 00000000 08000000 00000000 diff --git a/lld/test/ELF/emulation-amdgpu.s b/lld/test/ELF/emulation-amdgpu.s index 707f0aeb909efae47470a1d5492d65ad2b415318..2d9ae52548b41c3fc2f5738160c23ad7c5841032 100644 --- a/lld/test/ELF/emulation-amdgpu.s +++ b/lld/test/ELF/emulation-amdgpu.s @@ -1,10 +1,16 @@ # REQUIRES: amdgpu -# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa --amdhsa-code-object-version=4 %s -o %t.o # RUN: ld.lld %t.o -o %t -# RUN: llvm-readobj --file-headers %t | FileCheck %s +# RUN: llvm-readobj --file-headers %t | FileCheck --check-prefixes=CHECK,HSA4 %s # RUN: ld.lld -m elf64_amdgpu %t.o -o %t -# RUN: llvm-readobj --file-headers %t | FileCheck %s +# RUN: llvm-readobj --file-headers %t | FileCheck --check-prefixes=CHECK,HSA4 %s + +# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa --amdhsa-code-object-version=5 %s -o %t.o +# RUN: ld.lld %t.o -o %t +# RUN: llvm-readobj --file-headers %t | FileCheck --check-prefixes=CHECK,HSA5 %s +# RUN: ld.lld -m elf64_amdgpu %t.o -o %t +# RUN: llvm-readobj --file-headers %t | FileCheck --check-prefixes=CHECK,HSA5 %s # CHECK: ElfHeader { # CHECK-NEXT: Ident { @@ -13,8 +19,9 @@ # CHECK-NEXT: DataEncoding: LittleEndian (0x1) # CHECK-NEXT: FileVersion: 1 # CHECK-NEXT: OS/ABI: AMDGPU_HSA (0x40) -# CHECK-NEXT: ABIVersion: 2 -# CHECK-NEXT: Unused: (00 00 00 00 00 00 00) +# HSA4: ABIVersion: 2 +# HSA5: ABIVersion: 3 +# CHECK: Unused: (00 00 00 00 00 00 00) # CHECK-NEXT: } # CHECK-NEXT: Type: Executable (0x2) # CHECK-NEXT: Machine: EM_AMDGPU (0xE0) diff --git a/lld/test/ELF/i386-debug-noabs.test b/lld/test/ELF/i386-debug-noabs.test index 486cc8d9fbed82f8087f7d9d93213e453c5c96c7..02ef238dd148532f2854c2f3ce68a030cbd942fa 100644 --- a/lld/test/ELF/i386-debug-noabs.test +++ b/lld/test/ELF/i386-debug-noabs.test @@ -1,7 +1,8 @@ # REQUIRES: x86 # RUN: yaml2obj %s -o %t.o -# RUN: ld.lld %t.o -o /dev/null --entry 0 --fatal-warnings +# RUN: ld.lld %t.o -o /dev/null --entry 0 2>&1 | FileCheck %s +# CHECK: warning: {{.*}}:(.debug_info+0x41f): has non-ABS relocation R_386_GOTPC against symbol '_GLOBAL_OFFSET_TABLE_' ## This is for https://bugs.llvm.org//show_bug.cgi?id=34852. GCC 8.0 or ## earlier have a bug which creates non-absolute R_386_GOTPC relocations diff --git a/lld/test/ELF/non-abs-reloc.s b/lld/test/ELF/non-abs-reloc.s index 04f583ab253472da564973856a3eb0d31f25cff2..42b5f8fec1c436149a79b2953bb8bac021acf330 100644 --- a/lld/test/ELF/non-abs-reloc.s +++ b/lld/test/ELF/non-abs-reloc.s @@ -15,7 +15,8 @@ // DISASM-NEXT: 6: call{{.}} 0x5 /// There is currently no error for -r. See also https://github.com/ClangBuiltLinux/linux/issues/1937 -// RUN: ld.lld -T lds -r a.o -o /dev/null --fatal-warnings +// RUN: ld.lld -T lds -r a.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=REL-R --implicit-check-not=warning: +// REL-R: warning: {{.*}}:(.nonalloc1+0xa): has non-ABS relocation R_386_PC32 against symbol '' // RUN: llvm-mc -filetype=obj -triple=x86_64 asm -o b.o // RUN: ld.lld -T lds b.o -o b 2>&1 | FileCheck %s --check-prefix=CHECK2 --implicit-check-not=warning: diff --git a/lld/test/ELF/x86-64-dyn-rel-error.s b/lld/test/ELF/x86-64-dyn-rel-error.s index 8f41f1493dde7b278441ccb975b2aa6b6a77802b..edc2875c6fa63c7236178412dd24ab4e400172f0 100644 --- a/lld/test/ELF/x86-64-dyn-rel-error.s +++ b/lld/test/ELF/x86-64-dyn-rel-error.s @@ -1,17 +1,43 @@ -// REQUIRES: x86 -// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o -// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %p/Inputs/shared.s -o %t2.o -// RUN: ld.lld %t2.o -shared -o %t2.so -// RUN: not ld.lld -shared %t.o %t2.so -o /dev/null 2>&1 | FileCheck %s +# REQUIRES: x86 +# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple=x86_64 %p/Inputs/shared.s -o %t2.o +# RUN: ld.lld %t2.o -shared -o %t2.so --threads=1 +# RUN: not ld.lld -pie %t.o %t2.so -o /dev/null --threads=1 2>&1 | FileCheck %s +# RUN: not ld.lld -shared %t.o %t2.so -o /dev/null --threads=1 2>&1 | FileCheck %s --check-prefixes=CHECK,SHARED - .global _start -_start: - .data - .long zed +# CHECK: error: relocation R_X86_64_32 cannot be used against symbol 'zed'; recompile with -fPIC +# CHECK-NEXT: >>> defined in {{.*}}.so +# CHECK-NEXT: >>> referenced by {{.*}}.o:(.data+0x0) +# CHECK-EMPTY: +# CHECK-NEXT: error: relocation R_X86_64_PC32 cannot be used against symbol 'zed'; recompile with -fPIC +# CHECK-NEXT: >>> defined in {{.*}}.so +# CHECK-NEXT: >>> referenced by {{.*}}.o:(.data+0x4) +# CHECK-EMPTY: +# CHECK-NEXT: error: relocation R_X86_64_64 cannot be used against symbol '_start'; recompile with -fPIC +# SHARED: error: relocation R_X86_64_64 cannot be used against symbol 'main'; recompile with -fPIC +# SHARED: error: relocation R_X86_64_64 cannot be used against symbol 'data'; recompile with -fPIC +# CHECK-NOT: error: + +# RUN: ld.lld --noinhibit-exec %t.o %t2.so -o /dev/null 2>&1 | FileCheck --check-prefix=WARN %s +# RUN: not ld.lld --export-dynamic --unresolved-symbols=ignore-all %t.o -o /dev/null 2>&1 | FileCheck --check-prefix=WARN %s -// CHECK: error: relocation R_X86_64_32 cannot be used against symbol 'zed'; recompile with -fPIC +# WARN: relocation R_X86_64_32 cannot be used against symbol 'zed'; recompile with -fPIC +# WARN: relocation R_X86_64_PC32 cannot be used against symbol 'zed'; recompile with -fPIC + + .global _start, main, data + .type main, @function + .type data, @object +_start: + ret +main: + ret -// RUN: ld.lld --noinhibit-exec %t.o %t2.so -o /dev/null 2>&1 | FileCheck --check-prefix=WARN %s -// RUN: not ld.lld --export-dynamic --unresolved-symbols=ignore-all %t.o -o /dev/null 2>&1 | FileCheck --check-prefix=WARN %s +.data +data: +.long zed +.long zed - . -// WARN: relocation R_X86_64_32 cannot be used against symbol 'zed'; recompile with -fPIC +.rodata +.quad _start +.quad main +.quad data diff --git a/lld/test/ELF/x86-64-dyn-rel-error2.s b/lld/test/ELF/x86-64-dyn-rel-error2.s deleted file mode 100644 index 853e61faef68e57885448791efadb87e6e589aa4..0000000000000000000000000000000000000000 --- a/lld/test/ELF/x86-64-dyn-rel-error2.s +++ /dev/null @@ -1,14 +0,0 @@ -// REQUIRES: x86 -// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o -// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %p/Inputs/shared.s -o %t2.o -// RUN: ld.lld %t2.o -shared -o %t2.so -// RUN: not ld.lld -shared %t.o %t2.so -o /dev/null 2>&1 | FileCheck %s - -// CHECK: error: relocation R_X86_64_PC32 cannot be used against symbol 'zed'; recompile with -fPIC -// CHECK: >>> defined in {{.*}}.so -// CHECK: >>> referenced by {{.*}}.o:(.data+0x0) - - .global _start -_start: - .data - .long zed - . diff --git a/lld/test/ELF/x86-64-reloc-32.s b/lld/test/ELF/x86-64-reloc-32.s index 70a46301ad24a4b42508af6b712bf259cbfa463b..bf2fc26a5820486019e5fed664bc1fb0141566c0 100644 --- a/lld/test/ELF/x86-64-reloc-32.s +++ b/lld/test/ELF/x86-64-reloc-32.s @@ -1,9 +1,6 @@ # REQUIRES: x86 # RUN: rm -rf %t && split-file %s %t - -## Check recompile with -fPIC error message # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %t/shared.s -o %t/shared.o -# RUN: not ld.lld -shared %t/shared.o -o /dev/null 2>&1 | FileCheck %s # CHECK: error: relocation R_X86_64_32 cannot be used against symbol '_shared'; recompile with -fPIC # CHECK: >>> defined in {{.*}} diff --git a/lldb/include/lldb/Core/Address.h b/lldb/include/lldb/Core/Address.h index b19e694427546f8b1b07b54293023dc2e01569a8..725b5d9f91d3d5200195bbb2c23a9ba6e96e5328 100644 --- a/lldb/include/lldb/Core/Address.h +++ b/lldb/include/lldb/Core/Address.h @@ -14,6 +14,8 @@ #include "lldb/lldb-private-enumerations.h" #include "lldb/lldb-types.h" +#include "llvm/ADT/StringRef.h" + #include #include @@ -237,6 +239,12 @@ public: /// contains the address, otherwise dumping the range that contains the /// address. /// + /// \param[in] pattern + /// An optional regex pattern to match against the description. If + /// specified, parts of the description matching this pattern may be + /// highlighted or processed differently. If this parameter is an empty + /// string or not provided, no highlighting is applied. + /// /// \return /// Returns \b true if the address was able to be displayed. /// File and load addresses may be unresolved and it may not be @@ -246,8 +254,8 @@ public: /// \see Address::DumpStyle bool Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, DumpStyle fallback_style = DumpStyleInvalid, - uint32_t addr_byte_size = UINT32_MAX, - bool all_ranges = false) const; + uint32_t addr_byte_size = UINT32_MAX, bool all_ranges = false, + llvm::StringRef pattern = "") const; AddressClass GetAddressClass() const; diff --git a/lldb/include/lldb/Core/Debugger.h b/lldb/include/lldb/Core/Debugger.h index e4ee94809cf1a09f198af7cc1a5527a02c99d58c..c6d603ca5dcde06a0dd4f0c8d4b7d9df9231d715 100644 --- a/lldb/include/lldb/Core/Debugger.h +++ b/lldb/include/lldb/Core/Debugger.h @@ -321,6 +321,10 @@ public: llvm::StringRef GetAutosuggestionAnsiSuffix() const; + llvm::StringRef GetRegexMatchAnsiPrefix() const; + + llvm::StringRef GetRegexMatchAnsiSuffix() const; + bool GetShowDontUsePoHint() const; bool GetUseSourceCache() const; diff --git a/lldb/include/lldb/Symbol/Symbol.h b/lldb/include/lldb/Symbol/Symbol.h index 44a2d560010fe40327cb3bc387b6e4555de040a5..e6c0b495bcf28caa6ab5c118abef26dce065a211 100644 --- a/lldb/include/lldb/Symbol/Symbol.h +++ b/lldb/include/lldb/Symbol/Symbol.h @@ -174,8 +174,8 @@ public: void SetFlags(uint32_t flags) { m_flags = flags; } - void GetDescription(Stream *s, lldb::DescriptionLevel level, - Target *target) const; + void GetDescription(Stream *s, lldb::DescriptionLevel level, Target *target, + llvm::StringRef pattern = "") const; bool IsSynthetic() const { return m_is_synthetic; } diff --git a/lldb/include/lldb/Symbol/SymbolContext.h b/lldb/include/lldb/Symbol/SymbolContext.h index b0f5ffead2a16569893a9c8bb4bee85e9fe5e5ab..26f3bac09a96263e5192bac7b9ce9189a2ca0942 100644 --- a/lldb/include/lldb/Symbol/SymbolContext.h +++ b/lldb/include/lldb/Symbol/SymbolContext.h @@ -145,13 +145,19 @@ public: /// is dumped if this flag is \b true, otherwise the line info /// of the actual inlined function is dumped. /// + /// \param[in] pattern + /// An optional regex pattern to match against the stop context + /// description. If specified, parts of the description matching this + /// pattern may be highlighted or processed differently. If this parameter + /// is an empty string or not provided, no highlighting is applied. + /// /// \return /// \b true if some text was dumped, \b false otherwise. bool DumpStopContext(Stream *s, ExecutionContextScope *exe_scope, const Address &so_addr, bool show_fullpaths, bool show_module, bool show_inlined_frames, - bool show_function_arguments, - bool show_function_name) const; + bool show_function_arguments, bool show_function_name, + llvm::StringRef pattern = "") const; /// Get the address range contained within a symbol context. /// @@ -217,8 +223,8 @@ public: /// The symbol that was found, or \b nullptr if none was found. const Symbol *FindBestGlobalDataSymbol(ConstString name, Status &error); - void GetDescription(Stream *s, lldb::DescriptionLevel level, - Target *target) const; + void GetDescription(Stream *s, lldb::DescriptionLevel level, Target *target, + llvm::StringRef pattern = "") const; uint32_t GetResolvedMask() const; diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h index 4646e3070cf141f3915a04cf4b4bbe6e247ee334..24c599e044c78fa797ff7c13267beeda9aa1dd8c 100644 --- a/lldb/include/lldb/Target/Process.h +++ b/lldb/include/lldb/Target/Process.h @@ -558,7 +558,10 @@ public: /// /// Subclasses that override this method should always call this superclass /// method. - virtual void Finalize(); + /// If you are running Finalize in your Process subclass Destructor, pass + /// \b true. If we are in the destructor, shared_from_this will no longer + /// work, so we have to avoid doing anything that might trigger that. + virtual void Finalize(bool destructing); /// Return whether this object is valid (i.e. has not been finalized.) /// @@ -3079,6 +3082,11 @@ protected: /// This is set at the beginning of Process::Finalize() to stop functions /// from looking up or creating things during or after a finalize call. std::atomic m_finalizing; + // When we are "Finalizing" we need to do some cleanup. But if the Finalize + // call is coming in the Destructor, we can't do any actual work in the + // process because that is likely to call "shared_from_this" which crashes + // if run while destructing. We use this flag to determine that. + std::atomic m_destructing; /// Mask for code an data addresses. The default value (0) means no mask is /// set. The bits set to 1 indicate bits that are NOT significant for diff --git a/lldb/include/lldb/Utility/Stream.h b/lldb/include/lldb/Utility/Stream.h index 1a5fd343e4df0dc9b23dc9ffea5cee4d10302877..20c55ac4597ae614045d68595eaf9b381e65f243 100644 --- a/lldb/include/lldb/Utility/Stream.h +++ b/lldb/include/lldb/Utility/Stream.h @@ -231,6 +231,40 @@ public: /// The string to be output to the stream. size_t PutCString(llvm::StringRef cstr); + /// Output a C string to the stream with color highlighting. + /// + /// Print a C string \a text to the stream, applying color highlighting to + /// the portions of the string that match the regex pattern \a pattern. The + /// pattern is matched as many times as possible throughout the string. If \a + /// pattern is nullptr, then no highlighting is applied. + /// + /// The highlighting is applied by enclosing the matching text in ANSI color + /// codes. The \a prefix parameter specifies the ANSI code to start the color + /// (the standard value is assumed to be 'ansi.fg.red', representing red + /// foreground), and the \a suffix parameter specifies the ANSI code to end + /// the color (the standard value is assumed to be 'ansi.normal', resetting to + /// default text style). These constants should be defined appropriately in + /// your environment. + /// + /// \param[in] text + /// The string to be output to the stream. + /// + /// \param[in] pattern + /// The regex pattern to match against the \a text string. Portions of \a + /// text matching this pattern will be colorized. If this parameter is + /// nullptr, highlighting is not performed. + /// \param[in] prefix + /// The ANSI color code to start colorization. This is + /// environment-dependent. + /// \param[in] suffix + /// The ANSI color code to end colorization. This is + /// environment-dependent. + + void PutCStringColorHighlighted(llvm::StringRef text, + llvm::StringRef pattern = "", + llvm::StringRef prefix = "", + llvm::StringRef suffix = ""); + /// Output and End of Line character to the stream. size_t EOL(); diff --git a/lldb/source/Commands/CommandCompletions.cpp b/lldb/source/Commands/CommandCompletions.cpp index 4d7e3d7f2497bb117687c01ae523fae16b8394ae..0b69ce098195b119c229f3f699c3f4f555141d7d 100644 --- a/lldb/source/Commands/CommandCompletions.cpp +++ b/lldb/source/Commands/CommandCompletions.cpp @@ -333,7 +333,7 @@ static void DiskFilesOrDirectories(const llvm::Twine &partial_name, llvm::StringRef SearchDir; llvm::StringRef PartialItem; - if (CompletionBuffer.startswith("~")) { + if (CompletionBuffer.starts_with("~")) { llvm::StringRef Buffer = CompletionBuffer; size_t FirstSep = Buffer.find_if([](char c) { return path::is_separator(c); }); diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp index 58785cde3ec7c63cc46f7487d7603c465b28a49b..63232c221ad1db934641338136eccd893272140d 100644 --- a/lldb/source/Commands/CommandObjectTarget.cpp +++ b/lldb/source/Commands/CommandObjectTarget.cpp @@ -8,6 +8,7 @@ #include "CommandObjectTarget.h" +#include "lldb/Core/Address.h" #include "lldb/Core/Debugger.h" #include "lldb/Core/IOHandler.h" #include "lldb/Core/Module.h" @@ -1532,7 +1533,7 @@ static void DumpOsoFilesTable(Stream &strm, static void DumpAddress(ExecutionContextScope *exe_scope, const Address &so_addr, bool verbose, bool all_ranges, - Stream &strm) { + Stream &strm, llvm::StringRef pattern = "") { strm.IndentMore(); strm.Indent(" Address: "); so_addr.Dump(&strm, exe_scope, Address::DumpStyleModuleWithFileAddress); @@ -1542,13 +1543,14 @@ static void DumpAddress(ExecutionContextScope *exe_scope, strm.Indent(" Summary: "); const uint32_t save_indent = strm.GetIndentLevel(); strm.SetIndentLevel(save_indent + 13); - so_addr.Dump(&strm, exe_scope, Address::DumpStyleResolvedDescription); + so_addr.Dump(&strm, exe_scope, Address::DumpStyleResolvedDescription, + Address::DumpStyleInvalid, UINT32_MAX, false, pattern); strm.SetIndentLevel(save_indent); // Print out detailed address information when verbose is enabled if (verbose) { strm.EOL(); so_addr.Dump(&strm, exe_scope, Address::DumpStyleDetailedSymbolContext, - Address::DumpStyleInvalid, UINT32_MAX, all_ranges); + Address::DumpStyleInvalid, UINT32_MAX, all_ranges, pattern); } strm.IndentLess(); } @@ -1593,6 +1595,7 @@ static uint32_t LookupSymbolInModule(CommandInterpreter &interpreter, return 0; SymbolContext sc; + const bool use_color = interpreter.GetDebugger().GetUseColor(); std::vector match_indexes; ConstString symbol_name(name); uint32_t num_matches = 0; @@ -1618,12 +1621,19 @@ static uint32_t LookupSymbolInModule(CommandInterpreter &interpreter, if (symbol->ValueIsAddress()) { DumpAddress( interpreter.GetExecutionContext().GetBestExecutionContextScope(), - symbol->GetAddressRef(), verbose, all_ranges, strm); + symbol->GetAddressRef(), verbose, all_ranges, strm, + use_color && name_is_regex ? name : nullptr); strm.EOL(); } else { strm.IndentMore(); strm.Indent(" Name: "); - strm.PutCString(symbol->GetDisplayName().GetStringRef()); + llvm::StringRef ansi_prefix = + interpreter.GetDebugger().GetRegexMatchAnsiPrefix(); + llvm::StringRef ansi_suffix = + interpreter.GetDebugger().GetRegexMatchAnsiSuffix(); + strm.PutCStringColorHighlighted( + symbol->GetDisplayName().GetStringRef(), + use_color ? name : nullptr, ansi_prefix, ansi_suffix); strm.EOL(); strm.Indent(" Value: "); strm.Printf("0x%16.16" PRIx64 "\n", symbol->GetRawValue()); diff --git a/lldb/source/Core/Address.cpp b/lldb/source/Core/Address.cpp index 189d50fe962a651c78779c78df19b87c9f6ee7d6..19d34db44ea55cf2cea2ce8c24bc3aea9f51c73c 100644 --- a/lldb/source/Core/Address.cpp +++ b/lldb/source/Core/Address.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "lldb/Core/Address.h" +#include "lldb/Core/Debugger.h" #include "lldb/Core/Declaration.h" #include "lldb/Core/DumpDataExtractor.h" #include "lldb/Core/Module.h" @@ -28,6 +29,7 @@ #include "lldb/Target/Process.h" #include "lldb/Target/SectionLoadList.h" #include "lldb/Target/Target.h" +#include "lldb/Utility/AnsiTerminal.h" #include "lldb/Utility/ConstString.h" #include "lldb/Utility/DataExtractor.h" #include "lldb/Utility/Endian.h" @@ -405,7 +407,7 @@ bool Address::GetDescription(Stream &s, Target &target, bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, DumpStyle fallback_style, uint32_t addr_size, - bool all_ranges) const { + bool all_ranges, llvm::StringRef pattern) const { // If the section was nullptr, only load address is going to work unless we // are trying to deref a pointer SectionSP section_sp(GetSection()); @@ -501,7 +503,6 @@ bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, pointer_size = target->GetArchitecture().GetAddressByteSize(); else if (module_sp) pointer_size = module_sp->GetArchitecture().GetAddressByteSize(); - bool showed_info = false; if (section_sp) { SectionType sect_type = section_sp->GetType(); @@ -515,7 +516,16 @@ bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, if (symbol) { const char *symbol_name = symbol->GetName().AsCString(); if (symbol_name) { - s->PutCString(symbol_name); + llvm::StringRef ansi_prefix; + llvm::StringRef ansi_suffix; + if (target) { + ansi_prefix = + target->GetDebugger().GetRegexMatchAnsiPrefix(); + ansi_suffix = + target->GetDebugger().GetRegexMatchAnsiSuffix(); + } + s->PutCStringColorHighlighted(symbol_name, pattern, + ansi_prefix, ansi_suffix); addr_t delta = file_Addr - symbol->GetAddressRef().GetFileAddress(); if (delta) @@ -643,7 +653,7 @@ bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, pointer_sc.symbol != nullptr) { s->PutCString(": "); pointer_sc.DumpStopContext(s, exe_scope, so_addr, true, false, - false, true, true); + false, true, true, pattern); } } } @@ -682,19 +692,22 @@ bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, // address. sc.DumpStopContext(s, exe_scope, *this, show_fullpaths, show_module, show_inlined_frames, - show_function_arguments, show_function_name); + show_function_arguments, show_function_name, + pattern); } else { // We found a symbol but it was in a different section so it // isn't the symbol we should be showing, just show the section // name + offset - Dump(s, exe_scope, DumpStyleSectionNameOffset); + Dump(s, exe_scope, DumpStyleSectionNameOffset, DumpStyleInvalid, + UINT32_MAX, false, pattern); } } } } } else { if (fallback_style != DumpStyleInvalid) - return Dump(s, exe_scope, fallback_style, DumpStyleInvalid, addr_size); + return Dump(s, exe_scope, fallback_style, DumpStyleInvalid, addr_size, + false, pattern); return false; } break; @@ -715,7 +728,7 @@ bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, sc.symbol->GetAddressRef().GetSection() != GetSection()) sc.symbol = nullptr; } - sc.GetDescription(s, eDescriptionLevelBrief, target); + sc.GetDescription(s, eDescriptionLevelBrief, target, pattern); if (sc.block) { bool can_create = true; @@ -763,7 +776,8 @@ bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, } } else { if (fallback_style != DumpStyleInvalid) - return Dump(s, exe_scope, fallback_style, DumpStyleInvalid, addr_size); + return Dump(s, exe_scope, fallback_style, DumpStyleInvalid, addr_size, + false, pattern); return false; } break; diff --git a/lldb/source/Core/CoreProperties.td b/lldb/source/Core/CoreProperties.td index 0e0f468d3ecd76468e88b581850d4d2f2db6a5ca..8d81967bdb50a4534b1398f360a0cd7832cc2ebc 100644 --- a/lldb/source/Core/CoreProperties.td +++ b/lldb/source/Core/CoreProperties.td @@ -203,6 +203,14 @@ let Definition = "debugger" in { Global, DefaultStringValue<"${ansi.normal}">, Desc<"When displaying suggestion in a color-enabled terminal, use the ANSI terminal code specified in this format immediately after the suggestion.">; + def ShowRegexMatchAnsiPrefix: Property<"show-regex-match-ansi-prefix", "String">, + Global, + DefaultStringValue<"${ansi.fg.red}">, + Desc<"When displaying a regex match in a color-enabled terminal, use the ANSI terminal code specified in this format immediately before the match.">; + def ShowRegexMatchAnsiSuffix: Property<"show-regex-match-ansi-suffix", "String">, + Global, + DefaultStringValue<"${ansi.normal}">, + Desc<"When displaying a regex match in a color-enabled terminal, use the ANSI terminal code specified in this format immediately after the match.">; def ShowDontUsePoHint: Property<"show-dont-use-po-hint", "Boolean">, Global, DefaultTrue, diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp index 21f71e449ca5ed014ddfe3275832875bdfc9f57e..97311b4716ac2f9d821ef95ec87c5047877c6bf1 100644 --- a/lldb/source/Core/Debugger.cpp +++ b/lldb/source/Core/Debugger.cpp @@ -453,6 +453,18 @@ llvm::StringRef Debugger::GetAutosuggestionAnsiSuffix() const { idx, g_debugger_properties[idx].default_cstr_value); } +llvm::StringRef Debugger::GetRegexMatchAnsiPrefix() const { + const uint32_t idx = ePropertyShowRegexMatchAnsiPrefix; + return GetPropertyAtIndexAs( + idx, g_debugger_properties[idx].default_cstr_value); +} + +llvm::StringRef Debugger::GetRegexMatchAnsiSuffix() const { + const uint32_t idx = ePropertyShowRegexMatchAnsiSuffix; + return GetPropertyAtIndexAs( + idx, g_debugger_properties[idx].default_cstr_value); +} + bool Debugger::GetShowDontUsePoHint() const { const uint32_t idx = ePropertyShowDontUsePoHint; return GetPropertyAtIndexAs( @@ -930,7 +942,7 @@ void Debugger::Clear() { for (TargetSP target_sp : m_target_list.Targets()) { if (target_sp) { if (ProcessSP process_sp = target_sp->GetProcessSP()) - process_sp->Finalize(); + process_sp->Finalize(false /* not destructing */); target_sp->Destroy(); } } diff --git a/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp b/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp index 70bb9aa7a833c0832803ef255bf3b7979f670d60..de739acf5b2a52daabf14ab287fd8c15925677e9 100644 --- a/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp +++ b/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp @@ -164,7 +164,7 @@ ProcessKDP::~ProcessKDP() { // make sure all of the broadcaster cleanup goes as planned. If we destruct // this class, then Process::~Process() might have problems trying to fully // destroy the broadcaster. - Finalize(); + Finalize(true /* destructing */); } Status ProcessKDP::DoWillLaunch(Module *module) { diff --git a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp index aedc43a015ff12b2613718305f90100079fbd754..a4540de4acc45140a5946b5997f80a295c9f1151 100644 --- a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp +++ b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp @@ -108,7 +108,7 @@ ProcessElfCore::~ProcessElfCore() { // make sure all of the broadcaster cleanup goes as planned. If we destruct // this class, then Process::~Process() might have problems trying to fully // destroy the broadcaster. - Finalize(); + Finalize(true /* destructing */); } lldb::addr_t ProcessElfCore::AddAddressRangeFromLoadSegment( diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp index b04319703b9469a84415b5032431b469c9b6da25..d5e557b4b88c076b64c05e84b6a0ccd59fd1895d 100644 --- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp +++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp @@ -303,7 +303,7 @@ ProcessGDBRemote::~ProcessGDBRemote() { // make sure all of the broadcaster cleanup goes as planned. If we destruct // this class, then Process::~Process() might have problems trying to fully // destroy the broadcaster. - Finalize(); + Finalize(true /* destructing */); // The general Finalize is going to try to destroy the process and that // SHOULD shut down the async thread. However, if we don't kill it it will diff --git a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp index 9830a4b8599dff8c93376011627e67adb1ec7f62..a2ea19388b75f2057258e31495fff8ef43b2dbdb 100644 --- a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp +++ b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp @@ -123,7 +123,7 @@ ProcessMachCore::~ProcessMachCore() { // make sure all of the broadcaster cleanup goes as planned. If we destruct // this class, then Process::~Process() might have problems trying to fully // destroy the broadcaster. - Finalize(); + Finalize(true /* destructing */); } bool ProcessMachCore::CheckAddressForDyldOrKernel(lldb::addr_t addr, diff --git a/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp b/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp index 0d5ca42691d3d43fb51bb0c6da5d58811fd89776..b72307c7e4b96568c22792336230a2e74e66ba59 100644 --- a/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp +++ b/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp @@ -166,7 +166,7 @@ ProcessMinidump::~ProcessMinidump() { // make sure all of the broadcaster cleanup goes as planned. If we destruct // this class, then Process::~Process() might have problems trying to fully // destroy the broadcaster. - Finalize(); + Finalize(true /* destructing */); } void ProcessMinidump::Initialize() { diff --git a/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp b/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp index 54b367727913ca777de1d317f582cc2a7422ace3..66f861350d14d5700c6975d02f19e7a1fbde0396 100644 --- a/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp +++ b/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp @@ -140,7 +140,7 @@ ScriptedProcess::~ScriptedProcess() { // make sure all of the broadcaster cleanup goes as planned. If we destruct // this class, then Process::~Process() might have problems trying to fully // destroy the broadcaster. - Finalize(); + Finalize(true /* destructing */); } void ScriptedProcess::Initialize() { diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.cpp index 44e76022790130cedfdc1959d8b3428d2c98e69b..44421c0eda3eec9524855a26b3839a9de97d3701 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.cpp @@ -7,10 +7,27 @@ //===----------------------------------------------------------------------===// #include "DWARFDeclContext.h" +#include "llvm/Support/raw_ostream.h" using namespace lldb_private::dwarf; using namespace lldb_private::plugin::dwarf; +/// Returns the name of `entry` if it has one, or the appropriate "anonymous +/// {namespace, class, struct, union}". +static const char *GetName(DWARFDeclContext::Entry entry) { + if (entry.name != nullptr) + return entry.name; + if (entry.tag == DW_TAG_namespace) + return "(anonymous namespace)"; + if (entry.tag == DW_TAG_class_type) + return "(anonymous class)"; + if (entry.tag == DW_TAG_structure_type) + return "(anonymous struct)"; + if (entry.tag == DW_TAG_union_type) + return "(anonymous union)"; + return "(anonymous)"; +} + const char *DWARFDeclContext::GetQualifiedName() const { if (m_qualified_name.empty()) { // The declaration context array for a class named "foo" in namespace @@ -26,26 +43,10 @@ const char *DWARFDeclContext::GetQualifiedName() const { m_qualified_name.append(m_entries[0].name); } } else { - collection::const_reverse_iterator pos; - collection::const_reverse_iterator begin = m_entries.rbegin(); - collection::const_reverse_iterator end = m_entries.rend(); - for (pos = begin; pos != end; ++pos) { - if (pos != begin) - m_qualified_name.append("::"); - if (pos->name == nullptr) { - if (pos->tag == DW_TAG_namespace) - m_qualified_name.append("(anonymous namespace)"); - else if (pos->tag == DW_TAG_class_type) - m_qualified_name.append("(anonymous class)"); - else if (pos->tag == DW_TAG_structure_type) - m_qualified_name.append("(anonymous struct)"); - else if (pos->tag == DW_TAG_union_type) - m_qualified_name.append("(anonymous union)"); - else - m_qualified_name.append("(anonymous)"); - } else - m_qualified_name.append(pos->name); - } + llvm::raw_string_ostream string_stream(m_qualified_name); + llvm::interleave( + llvm::reverse(m_entries), string_stream, + [&](auto entry) { string_stream << GetName(entry); }, "::"); } } } diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index d4cc26a3c329be924f196a00750fa46180cdee3d..d4c573ecd468c416c1f976f775547b304dca591a 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -132,6 +132,11 @@ public: } // namespace +bool IsStructOrClassTag(llvm::dwarf::Tag Tag) { + return Tag == llvm::dwarf::Tag::DW_TAG_class_type || + Tag == llvm::dwarf::Tag::DW_TAG_structure_type; +} + static PluginProperties &GetGlobalPluginProperties() { static PluginProperties g_settings; return g_settings; @@ -2947,29 +2952,18 @@ TypeSP SymbolFileDWARF::FindCompleteObjCDefinitionTypeForDIE( m_index->GetCompleteObjCClass( type_name, must_be_implementation, [&](DWARFDIE type_die) { - bool try_resolving_type = false; - // Don't try and resolve the DIE we are looking for with the DIE // itself! - if (type_die != die) { - switch (type_die.Tag()) { - case DW_TAG_class_type: - case DW_TAG_structure_type: - try_resolving_type = true; - break; - default: - break; - } - } - if (!try_resolving_type) + if (type_die == die || !IsStructOrClassTag(type_die.Tag())) return true; if (must_be_implementation && - type_die.Supports_DW_AT_APPLE_objc_complete_type()) - try_resolving_type = type_die.GetAttributeValueAsUnsigned( + type_die.Supports_DW_AT_APPLE_objc_complete_type()) { + const bool try_resolving_type = type_die.GetAttributeValueAsUnsigned( DW_AT_APPLE_objc_complete_type, 0); - if (!try_resolving_type) - return true; + if (!try_resolving_type) + return true; + } Type *resolved_type = ResolveType(type_die, false, true); if (!resolved_type || resolved_type == DIE_IS_BEING_PARSED) @@ -3128,36 +3122,12 @@ SymbolFileDWARF::FindDefinitionTypeForDWARFDeclContext(const DWARFDIE &die) { if (type_system && !type_system->SupportsLanguage(GetLanguage(*type_die.GetCU()))) return true; - bool try_resolving_type = false; - // Don't try and resolve the DIE we are looking for with the DIE - // itself! const dw_tag_t type_tag = type_die.Tag(); - // Make sure the tags match - if (type_tag == tag) { - // The tags match, lets try resolving this type - try_resolving_type = true; - } else { - // The tags don't match, but we need to watch our for a forward - // declaration for a struct and ("struct foo") ends up being a - // class ("class foo { ... };") or vice versa. - switch (type_tag) { - case DW_TAG_class_type: - // We had a "class foo", see if we ended up with a "struct foo - // { ... };" - try_resolving_type = (tag == DW_TAG_structure_type); - break; - case DW_TAG_structure_type: - // We had a "struct foo", see if we ended up with a "class foo - // { ... };" - try_resolving_type = (tag == DW_TAG_class_type); - break; - default: - // Tags don't match, don't event try to resolve using this type - // whose name matches.... - break; - } - } + // Resolve the type if both have the same tag or {class, struct} tags. + const bool try_resolving_type = + type_tag == tag || + (IsStructOrClassTag(type_tag) && IsStructOrClassTag(tag)); if (!try_resolving_type) { if (log) { diff --git a/lldb/source/Symbol/Symbol.cpp b/lldb/source/Symbol/Symbol.cpp index 26b4c4d62ad9c24667c1279c19983324da7a7aff..fcc45f861c2255ba507997fbcbc71e6633930166 100644 --- a/lldb/source/Symbol/Symbol.cpp +++ b/lldb/source/Symbol/Symbol.cpp @@ -8,6 +8,8 @@ #include "lldb/Symbol/Symbol.h" +#include "lldb/Core/Address.h" +#include "lldb/Core/Debugger.h" #include "lldb/Core/Module.h" #include "lldb/Core/ModuleSpec.h" #include "lldb/Core/Section.h" @@ -225,7 +227,7 @@ bool Symbol::IsTrampoline() const { return m_type == eSymbolTypeTrampoline; } bool Symbol::IsIndirect() const { return m_type == eSymbolTypeResolver; } void Symbol::GetDescription(Stream *s, lldb::DescriptionLevel level, - Target *target) const { + Target *target, llvm::StringRef pattern) const { s->Printf("id = {0x%8.8x}", m_uid); if (m_addr_range.GetBaseAddress().GetSection()) { @@ -252,11 +254,24 @@ void Symbol::GetDescription(Stream *s, lldb::DescriptionLevel level, s->Printf(", value = 0x%16.16" PRIx64, m_addr_range.GetBaseAddress().GetOffset()); } - ConstString demangled = GetMangled().GetDemangledName(); - if (demangled) - s->Printf(", name=\"%s\"", demangled.AsCString()); - if (m_mangled.GetMangledName()) - s->Printf(", mangled=\"%s\"", m_mangled.GetMangledName().AsCString()); + llvm::StringRef ansi_prefix; + llvm::StringRef ansi_suffix; + if (target) { + ansi_prefix = target->GetDebugger().GetRegexMatchAnsiPrefix(); + ansi_suffix = target->GetDebugger().GetRegexMatchAnsiSuffix(); + } + if (ConstString demangled = m_mangled.GetDemangledName()) { + s->PutCString(", name=\""); + s->PutCStringColorHighlighted(demangled.GetStringRef(), pattern, + ansi_prefix, ansi_suffix); + s->PutCString("\""); + } + if (ConstString mangled_name = m_mangled.GetMangledName()) { + s->PutCString(", mangled=\""); + s->PutCStringColorHighlighted(mangled_name.GetStringRef(), pattern, + ansi_prefix, ansi_suffix); + s->PutCString("\""); + } } void Symbol::Dump(Stream *s, Target *target, uint32_t index, diff --git a/lldb/source/Symbol/SymbolContext.cpp b/lldb/source/Symbol/SymbolContext.cpp index 63968ec2d150670531381617c51d2e57b0cc6fdd..9fd40b5ca567f874c670ed5517ed2f3e47f66fc3 100644 --- a/lldb/source/Symbol/SymbolContext.cpp +++ b/lldb/source/Symbol/SymbolContext.cpp @@ -8,6 +8,7 @@ #include "lldb/Symbol/SymbolContext.h" +#include "lldb/Core/Address.h" #include "lldb/Core/Debugger.h" #include "lldb/Core/Module.h" #include "lldb/Core/ModuleSpec.h" @@ -71,7 +72,8 @@ bool SymbolContext::DumpStopContext(Stream *s, ExecutionContextScope *exe_scope, const Address &addr, bool show_fullpaths, bool show_module, bool show_inlined_frames, bool show_function_arguments, - bool show_function_name) const { + bool show_function_name, + llvm::StringRef pattern) const { bool dumped_something = false; if (show_module && module_sp) { if (show_fullpaths) @@ -81,7 +83,6 @@ bool SymbolContext::DumpStopContext(Stream *s, ExecutionContextScope *exe_scope, s->PutChar('`'); dumped_something = true; } - if (function != nullptr) { SymbolContext inline_parent_sc; Address inline_parent_addr; @@ -94,8 +95,16 @@ bool SymbolContext::DumpStopContext(Stream *s, ExecutionContextScope *exe_scope, name = function->GetNameNoArguments(); if (!name) name = function->GetName(); - if (name) - name.Dump(s); + if (name) { + llvm::StringRef ansi_prefix; + llvm::StringRef ansi_suffix; + if (target_sp) { + ansi_prefix = target_sp->GetDebugger().GetRegexMatchAnsiPrefix(); + ansi_suffix = target_sp->GetDebugger().GetRegexMatchAnsiSuffix(); + } + s->PutCStringColorHighlighted(name.GetStringRef(), pattern, ansi_prefix, + ansi_suffix); + } } if (addr.IsValid()) { @@ -163,7 +172,14 @@ bool SymbolContext::DumpStopContext(Stream *s, ExecutionContextScope *exe_scope, dumped_something = true; if (symbol->GetType() == eSymbolTypeTrampoline) s->PutCString("symbol stub for: "); - symbol->GetName().Dump(s); + llvm::StringRef ansi_prefix; + llvm::StringRef ansi_suffix; + if (target_sp) { + ansi_prefix = target_sp->GetDebugger().GetRegexMatchAnsiPrefix(); + ansi_suffix = target_sp->GetDebugger().GetRegexMatchAnsiSuffix(); + } + s->PutCStringColorHighlighted(symbol->GetName().GetStringRef(), pattern, + ansi_prefix, ansi_suffix); } if (addr.IsValid() && symbol->ValueIsAddress()) { @@ -186,7 +202,8 @@ bool SymbolContext::DumpStopContext(Stream *s, ExecutionContextScope *exe_scope, } void SymbolContext::GetDescription(Stream *s, lldb::DescriptionLevel level, - Target *target) const { + Target *target, + llvm::StringRef pattern) const { if (module_sp) { s->Indent(" Module: file = \""); module_sp->GetFileSpec().Dump(s->AsRawOstream()); @@ -246,7 +263,7 @@ void SymbolContext::GetDescription(Stream *s, lldb::DescriptionLevel level, if (symbol != nullptr) { s->Indent(" Symbol: "); - symbol->GetDescription(s, level, target); + symbol->GetDescription(s, level, target, pattern); s->EOL(); } diff --git a/lldb/source/Symbol/Variable.cpp b/lldb/source/Symbol/Variable.cpp index 85ceadd20c611eadd2171b51c9ebd2fbc7f91760..db740cb7cb6e410285eb18f680c77f163c9f89c5 100644 --- a/lldb/source/Symbol/Variable.cpp +++ b/lldb/source/Symbol/Variable.cpp @@ -227,7 +227,8 @@ bool Variable::LocationIsValidForFrame(StackFrame *frame) { // contains the current address when converted to a load address return m_location_list.ContainsAddress( loclist_base_load_addr, - frame->GetFrameCodeAddress().GetLoadAddress(target_sp.get())); + frame->GetFrameCodeAddressForSymbolication().GetLoadAddress( + target_sp.get())); } } return false; diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp index 2d77144a9b28df01f1edf8ef65bf319c85c3e2cd..aa3b04c43cc5cdec2da008f3142c4b8a97ff66dc 100644 --- a/lldb/source/Target/Process.cpp +++ b/lldb/source/Target/Process.cpp @@ -445,7 +445,7 @@ Process::Process(lldb::TargetSP target_sp, ListenerSP listener_sp, m_memory_cache(*this), m_allocated_memory_cache(*this), m_should_detach(false), m_next_event_action_up(), m_public_run_lock(), m_private_run_lock(), m_currently_handling_do_on_removals(false), - m_resume_requested(false), m_finalizing(false), + m_resume_requested(false), m_finalizing(false), m_destructing(false), m_clear_thread_plans_on_stop(false), m_force_next_event_delivery(false), m_last_broadcast_state(eStateInvalid), m_destroy_in_process(false), m_can_interpret_function_calls(false), m_run_thread_plan_lock(), @@ -518,9 +518,11 @@ ProcessProperties &Process::GetGlobalProperties() { return *g_settings_ptr; } -void Process::Finalize() { +void Process::Finalize(bool destructing) { if (m_finalizing.exchange(true)) return; + if (destructing) + m_destructing.exchange(true); // Destroy the process. This will call the virtual function DoDestroy under // the hood, giving our derived class a chance to do the ncessary tear down. @@ -1415,7 +1417,13 @@ bool Process::StateChangedIsHijackedForSynchronousResume() { StateType Process::GetPrivateState() { return m_private_state.GetValue(); } void Process::SetPrivateState(StateType new_state) { - if (m_finalizing) + // Use m_destructing not m_finalizing here. If we are finalizing a process + // that we haven't started tearing down, we'd like to be able to nicely + // detach if asked, but that requires the event system be live. That will + // not be true for an in-the-middle-of-being-destructed Process, since the + // event system relies on Process::shared_from_this, which may have already + // been destroyed. + if (m_destructing) return; Log *log(GetLog(LLDBLog::State | LLDBLog::Process | LLDBLog::Unwind)); diff --git a/lldb/source/Target/ProcessTrace.cpp b/lldb/source/Target/ProcessTrace.cpp index 061af9e0e520f1c8f0a71f8d4ac11191214eea83..6e5ef6a379f904245630a5edc5b6e93b2f5fa3dc 100644 --- a/lldb/source/Target/ProcessTrace.cpp +++ b/lldb/source/Target/ProcessTrace.cpp @@ -50,7 +50,7 @@ ProcessTrace::~ProcessTrace() { // make sure all of the broadcaster cleanup goes as planned. If we destruct // this class, then Process::~Process() might have problems trying to fully // destroy the broadcaster. - Finalize(); + Finalize(true /* destructing */); } void ProcessTrace::DidAttach(ArchSpec &process_arch) { diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp index 2e8d1dfdaa1769a53ff38f88b78b3a748bc54961..302c2bad7021b9f01cc0b7fe0ae439b18b01f802 100644 --- a/lldb/source/Target/Target.cpp +++ b/lldb/source/Target/Target.cpp @@ -197,7 +197,7 @@ void Target::DeleteCurrentProcess() { if (m_process_sp->IsAlive()) m_process_sp->Destroy(false); - m_process_sp->Finalize(); + m_process_sp->Finalize(false /* not destructing */); CleanupProcess(); diff --git a/lldb/source/Utility/Stream.cpp b/lldb/source/Utility/Stream.cpp index af28a49a1f0c2b5ab705c1f2e2979d7e93279a38..62e061e9d09c077019f0dc0b5ff9cb06706850dc 100644 --- a/lldb/source/Utility/Stream.cpp +++ b/lldb/source/Utility/Stream.cpp @@ -8,11 +8,13 @@ #include "lldb/Utility/Stream.h" +#include "lldb/Utility/AnsiTerminal.h" #include "lldb/Utility/Endian.h" #include "lldb/Utility/VASPrintf.h" #include "llvm/ADT/SmallString.h" #include "llvm/Support/Format.h" #include "llvm/Support/LEB128.h" +#include "llvm/Support/Regex.h" #include @@ -70,6 +72,34 @@ size_t Stream::PutCString(llvm::StringRef str) { return bytes_written; } +void Stream::PutCStringColorHighlighted(llvm::StringRef text, + llvm::StringRef pattern, + llvm::StringRef prefix, + llvm::StringRef suffix) { + // Only apply color formatting when a pattern is present and both prefix and + // suffix are specified. In the absence of these conditions, output the text + // without color formatting. + if (pattern.empty() || (prefix.empty() && suffix.empty())) { + PutCString(text); + return; + } + + llvm::Regex reg_pattern(pattern); + llvm::SmallVector matches; + llvm::StringRef remaining = text; + std::string format_str = lldb_private::ansi::FormatAnsiTerminalCodes( + prefix.str() + "%.*s" + suffix.str()); + while (reg_pattern.match(remaining, &matches)) { + llvm::StringRef match = matches[0]; + size_t match_start_pos = match.data() - remaining.data(); + PutCString(remaining.take_front(match_start_pos)); + Printf(format_str.c_str(), match.size(), match.data()); + remaining = remaining.drop_front(match_start_pos + match.size()); + } + if (remaining.size()) + PutCString(remaining); +} + // Print a double quoted NULL terminated C string to the stream using the // printf format in "format". void Stream::QuotedCString(const char *cstr, const char *format) { diff --git a/lldb/test/API/driver/quit_speed/Makefile b/lldb/test/API/driver/quit_speed/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..10495940055b63d2b69fd0ee465e69dad1889d2f --- /dev/null +++ b/lldb/test/API/driver/quit_speed/Makefile @@ -0,0 +1,3 @@ +C_SOURCES := main.c + +include Makefile.rules diff --git a/lldb/test/API/driver/quit_speed/TestQuitWithProcess.py b/lldb/test/API/driver/quit_speed/TestQuitWithProcess.py new file mode 100644 index 0000000000000000000000000000000000000000..957586d41f6b405d44d78a2af7a5fdf7f4649565 --- /dev/null +++ b/lldb/test/API/driver/quit_speed/TestQuitWithProcess.py @@ -0,0 +1,34 @@ +""" +Test that killing the target while quitting doesn't stall +""" + + +import lldb +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil +import pexpect +from lldbsuite.test.lldbpexpect import PExpectTest + + +class DriverQuitSpeedTest(PExpectTest): + source = "main.c" + + def test_run_quit(self): + """Test that the lldb driver's batch mode works correctly.""" + self.build() + + exe = self.getBuildArtifact("a.out") + + # Turn on auto-confirm removes the wait for the prompt. + self.launch(executable=exe, extra_args=["-O", "settings set auto-confirm 1"]) + child = self.child + + # Launch the process without a TTY so we don't have to interrupt: + child.sendline("process launch -n") + print("launched process") + child.expect("Process ([\d]*) launched:") + print("Got launch message") + child.sendline("quit") + print("sent quit") + child.expect(pexpect.EOF, timeout=15) diff --git a/lldb/test/API/driver/quit_speed/main.c b/lldb/test/API/driver/quit_speed/main.c new file mode 100644 index 0000000000000000000000000000000000000000..3d6d45ce5e8068c7a419230cc17e9c53a5543616 --- /dev/null +++ b/lldb/test/API/driver/quit_speed/main.c @@ -0,0 +1,8 @@ +#include + +int main (int argc, char **argv) { + while(1) + usleep(5); + + return 0; +} diff --git a/lldb/test/API/functionalities/location-list-lookup/Makefile b/lldb/test/API/functionalities/location-list-lookup/Makefile index 78b0b11cb7484e55c4ce5e229ab2b06c0b3b9348..8e453681d7b390b13814b41ba77ec473d6e0ac90 100644 --- a/lldb/test/API/functionalities/location-list-lookup/Makefile +++ b/lldb/test/API/functionalities/location-list-lookup/Makefile @@ -1,3 +1,3 @@ -C_SOURCES := main.c +CXX_SOURCES := main.cpp CFLAGS_EXTRAS := -O1 include Makefile.rules diff --git a/lldb/test/API/functionalities/location-list-lookup/TestLocationListLookup.py b/lldb/test/API/functionalities/location-list-lookup/TestLocationListLookup.py index 4793447c59413290f2545bdd00ff7e9acad5d3a5..feea14ff355e829d7858a3698c6d02aa69ba21e3 100644 --- a/lldb/test/API/functionalities/location-list-lookup/TestLocationListLookup.py +++ b/lldb/test/API/functionalities/location-list-lookup/TestLocationListLookup.py @@ -7,16 +7,8 @@ from lldbsuite.test import lldbutil class LocationListLookupTestCase(TestBase): - def setUp(self): - # Call super's setUp(). - TestBase.setUp(self) - - @skipIf(oslist=["linux"], archs=["arm"]) - def test_loclist(self): - self.build() + def launch(self) -> lldb.SBProcess: exe = self.getBuildArtifact("a.out") - - # Create a target by the debugger. target = self.dbg.CreateTarget(exe) self.assertTrue(target, VALID_TARGET) self.dbg.SetAsync(False) @@ -27,12 +19,32 @@ class LocationListLookupTestCase(TestBase): self.assertTrue(process.IsValid()) self.assertTrue(process.is_stopped) - # Find `main` on the stack, then - # find `argv` local variable, then - # check that we can read the c-string in argv[0] + return process + + def check_local_vars(self, process: lldb.SBProcess, check_expr: bool): + # Find `bar` on the stack, then + # make sure we can read out the local + # variables (with both `frame var` and `expr`) for f in process.GetSelectedThread().frames: - if f.GetDisplayFunctionName() == "main": + frame_name = f.GetDisplayFunctionName() + if frame_name is not None and frame_name.startswith("Foo::bar"): argv = f.GetValueForVariablePath("argv").GetChildAtIndex(0) strm = lldb.SBStream() argv.GetDescription(strm) self.assertNotEqual(strm.GetData().find("a.out"), -1) + + if check_expr: + process.GetSelectedThread().SetSelectedFrame(f.idx) + self.expect_expr("this", result_type="Foo *") + + @skipIf(oslist=["linux"], archs=["arm"]) + @skipIfDarwin + def test_loclist_frame_var(self): + self.build() + self.check_local_vars(self.launch(), check_expr=False) + + @skipIf(compiler="clang", compiler_version=["<=", "11.0"]) + @skipUnlessDarwin + def test_loclist_expr(self): + self.build() + self.check_local_vars(self.launch(), check_expr=True) diff --git a/lldb/test/API/functionalities/location-list-lookup/main.c b/lldb/test/API/functionalities/location-list-lookup/main.c deleted file mode 100644 index 852772ee52ca2d9b77295f5f45e1378659c9acfd..0000000000000000000000000000000000000000 --- a/lldb/test/API/functionalities/location-list-lookup/main.c +++ /dev/null @@ -1,23 +0,0 @@ -#include -#include - -// The goal with this test is: -// 1. Have main() followed by foo() -// 2. Have the no-return call to abort() in main be the last instruction -// 3. Have the next instruction be the start of foo() -// 4. The debug info for argv uses a location list. -// clang at -O1 on x86_64 or arm64 has debuginfo like -// DW_AT_location (0x00000049: -// [0x0000000100003f15, 0x0000000100003f25): DW_OP_reg4 RSI -// [0x0000000100003f25, 0x0000000100003f5b): DW_OP_reg15 R15) - -void foo(int); -int main(int argc, char **argv) { - char *file = argv[0]; - char f0 = file[0]; - printf("%c\n", f0); - foo(f0); - printf("%s %d\n", argv[0], argc); - abort(); /// argv is still be accessible here -} -void foo(int in) { printf("%d\n", in); } diff --git a/lldb/test/API/functionalities/location-list-lookup/main.cpp b/lldb/test/API/functionalities/location-list-lookup/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4ccdadbddbb555a06e48195d17c11280899d8af3 --- /dev/null +++ b/lldb/test/API/functionalities/location-list-lookup/main.cpp @@ -0,0 +1,23 @@ +#include +#include + +void func(int in); + +struct Foo { + int x; + [[clang::noinline]] void bar(char **argv); +}; + +int main(int argc, char **argv) { + Foo f{.x = 5}; + std::printf("%p\n", &f.x); + f.bar(argv); + return f.x; +} + +void Foo::bar(char **argv) { + std::printf("%p %p\n", argv, this); + std::abort(); /// 'this' should be still accessible +} + +void func(int in) { printf("%d\n", in); } diff --git a/lldb/test/API/lang/cpp/const_static_integral_member/TestConstStaticIntegralMember.py b/lldb/test/API/lang/cpp/const_static_integral_member/TestConstStaticIntegralMember.py index 60e116b4220778b966bce7cacf6655e35dcc393c..cdade2d335df0b17969df078d7dbbdbf3d9846a8 100644 --- a/lldb/test/API/lang/cpp/const_static_integral_member/TestConstStaticIntegralMember.py +++ b/lldb/test/API/lang/cpp/const_static_integral_member/TestConstStaticIntegralMember.py @@ -104,7 +104,7 @@ class TestCase(TestBase): # dsymutil strips the debug info for classes that only have const static # data members without locations. - @expectedFailureAll(debug_info=["dsym"]) + @expectedFailureAll(debug_info=["dsym"], dwarf_version=["<", "5"]) def test_class_with_only_const_static(self): self.build() lldbutil.run_to_source_breakpoint( @@ -120,6 +120,9 @@ class TestCase(TestBase): self.assertEqual(varobj.type.name, expect_type) self.assertEqual(varobj.value, expect_val) + @expectedFailureAll(dwarf_version=["<", "5"]) + # On linux this passes due to the manual index + @expectedFailureDarwin(debug_info=no_match(["dsym"])) def test_inline_static_members(self): self.build() lldbutil.run_to_source_breakpoint( @@ -167,6 +170,9 @@ class TestCase(TestBase): "ClassWithEnumAlias::enum_alias_alias", result_value="scoped_enum_case1" ) + @expectedFailureAll(dwarf_version=["<", "5"]) + # On linux this passes due to the manual index + @expectedFailureDarwin(debug_info=no_match(["dsym"])) def test_shadowed_static_inline_members(self): """Tests that the expression evaluator and SBAPI can both correctly determine the requested inline static variable diff --git a/lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py b/lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py index d2a21ad3cd1d44ff252742455b83ed873e581400..9b0755eea7d3ec5fcd9b717cf2e7d950faf98908 100644 --- a/lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py +++ b/lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py @@ -4,8 +4,8 @@ Test lldb-dap setBreakpoints request import os -import lldbdap_testcase import dap_server +import lldbdap_testcase from lldbsuite.test import lldbutil from lldbsuite.test.decorators import * from lldbsuite.test.lldbtest import * @@ -152,7 +152,13 @@ class TestDAP_variables(lldbdap_testcase.DAPTestCaseBase): globals = self.dap_server.get_global_variables() buffer_children = make_buffer_verify_dict(0, 32) verify_locals = { - "argc": {"equals": {"type": "int", "value": "1"}}, + "argc": { + "equals": {"type": "int", "value": "1"}, + "declaration": { + "equals": {"line": 12, "column": 14}, + "contains": {"path": ["lldb-dap", "variables", "main.cpp"]}, + }, + }, "argv": { "equals": {"type": "const char **"}, "startswith": {"value": "0x"}, diff --git a/lldb/test/Shell/Commands/command-image-lookup-color.test b/lldb/test/Shell/Commands/command-image-lookup-color.test new file mode 100644 index 0000000000000000000000000000000000000000..186526b7efaee51c91f65da889a10181a00518ec --- /dev/null +++ b/lldb/test/Shell/Commands/command-image-lookup-color.test @@ -0,0 +1,60 @@ +# RUN: %clang_host -g %S/Inputs/main.c -o %t + +# The file main.c is does not show up in search on Windows. +# UNSUPPORTED: system-windows + +# Until we figure out the correct CHECK lines. +# UNSUPPORTED: system-darwin + +# Checking simple regex search + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'image lookup -r -s ma' | FileCheck %s --check-prefix CHECK1 +# CHECK1: Name: {{.+}}31mma{{.+}}0min.c + +# Checking complex regex searches + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'image lookup -r -s main.c|foo' | FileCheck %s --check-prefix CHECK2 +# CHECK2: Name: {{.+}}31mmain.c{{.+}}0m + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'image lookup -r -s m[abc]' | FileCheck %s --check-prefix CHECK3 +# CHECK3: Name: {{.+}}31mma{{.+}}0min.c + +# Checking to ensure that no attempt is made to color anything when there are no matching symbols found + +# RUN: %lldb %t -o 'settings set use-color true' -o 'image lookup -r -s IMPPATTERN123456' | FileCheck %s --check-prefix CHECK4 +# CHECK4-NOT: {{[0-9]+}} symbols match the regular expression + +# Checking multiple matches on same symbol + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'image lookup -r -s (ma|n)' | FileCheck %s --check-prefix CHECK5 +# CHECK5: Name: {{.+}}31mma{{.+}}0mi{{.+}}31mn{{.+}}0m.c + +# Checking no colorization without regex search + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'image lookup -s main' | FileCheck %s --check-prefix CHECK6 +# CHECK6: Summary: {{.+}}`main at main.c: + +# Checking no colorization when use-color is false + +# RUN: %lldb %t -b -o 'settings set use-color false' -o 'image lookup -r -s ma' | FileCheck %s --check-prefix CHECK7 +# CHECK7: Name: main.c + +# Checking for custom colors + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'settings set show-regex-match-ansi-prefix ${ansi.fg.green}' -o 'image lookup -r -s ma' | FileCheck %s --check-prefix CHECK8 +# CHECK8: Name: {{.+}}32mma{{.+}}0min.c + +# Checking for functionality when there's prefix but no suffix + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'settings set show-regex-match-ansi-prefix ${ansi.fg.red}' -o 'settings set show-regex-match-ansi-suffix ""' -o 'image lookup -r -s ma' | FileCheck %s --check-prefix CHECK9 +# CHECK9: Name: {{.+}}31mmain.c + +# Checking for functionality when there's suffix but no prefix + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'settings set show-regex-match-ansi-prefix ""' -o 'settings set show-regex-match-ansi-suffix ${ansi.fg.red}' -o 'image lookup -r -s ma' | FileCheck %s --check-prefix CHECK10 +# CHECK10: Name: ma{{.+}}31min.c + +# Checking for no colorization when there's neither suffix nor prefix + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'settings set show-regex-match-ansi-prefix ""' -o 'settings set show-regex-match-ansi-suffix ""' -o 'image lookup -r -s ma' | FileCheck %s --check-prefix CHECK11 +# CHECK11: Name: main.c diff --git a/lldb/tools/lldb-dap/JSONUtils.cpp b/lldb/tools/lldb-dap/JSONUtils.cpp index 3a63046d9a888bf8f83a22e1b3836a13d69a4705..c8e5304ecec81a07b1bd9169542194eeb9e4ae4f 100644 --- a/lldb/tools/lldb-dap/JSONUtils.cpp +++ b/lldb/tools/lldb-dap/JSONUtils.cpp @@ -804,9 +804,11 @@ llvm::json::Value CreateStackFrame(lldb::SBFrame &frame) { llvm::raw_string_ostream os(frame_name); os << llvm::format_hex(frame.GetPC(), 18); } - bool is_optimized = frame.GetFunction().GetIsOptimized(); - if (is_optimized) + + // We only include `[opt]` if a custom frame format is not specified. + if (!g_dap.frame_format && frame.GetFunction().GetIsOptimized()) frame_name += " [opt]"; + EmplaceSafeString(object, "name", frame_name); auto source = CreateSource(frame); @@ -1101,6 +1103,29 @@ std::string CreateUniqueVariableNameForDisplay(lldb::SBValue v, // can use this optional information to present the // children in a paged UI and fetch them in chunks." // } +// "declaration": { +// "type": "object | undefined", +// "description": "Extension to the protocol that indicates the source +// location where the variable was declared. This value +// might not be present if no declaration is available.", +// "properties": { +// "path": { +// "type": "string | undefined", +// "description": "The source file path where the variable was +// declared." +// }, +// "line": { +// "type": "number | undefined", +// "description": "The 1-indexed source line where the variable was +// declared." +// }, +// "column": { +// "type": "number | undefined", +// "description": "The 1-indexed source column where the variable was +// declared." +// } +// } +// } // }, // "required": [ "name", "value", "variablesReference" ] // } @@ -1165,6 +1190,24 @@ llvm::json::Value CreateVariable(lldb::SBValue v, int64_t variablesReference, const char *evaluateName = evaluateStream.GetData(); if (evaluateName && evaluateName[0]) EmplaceSafeString(object, "evaluateName", std::string(evaluateName)); + + if (lldb::SBDeclaration decl = v.GetDeclaration(); decl.IsValid()) { + llvm::json::Object decl_obj; + if (lldb::SBFileSpec file = decl.GetFileSpec(); file.IsValid()) { + char path[PATH_MAX] = ""; + if (file.GetPath(path, sizeof(path)) && + lldb::SBFileSpec::ResolvePath(path, path, PATH_MAX)) { + decl_obj.try_emplace("path", std::string(path)); + } + } + + if (int line = decl.GetLine()) + decl_obj.try_emplace("line", line); + if (int column = decl.GetColumn()) + decl_obj.try_emplace("column", column); + + object.try_emplace("declaration", std::move(decl_obj)); + } return llvm::json::Value(std::move(object)); } diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt index 1c983165b2ef0039ebc256b6daf67f431da3e4b7..35be9bf16b5e1e634a058e2b634c1f9fa6cc755b 100644 --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -581,7 +581,9 @@ set(LLVM_TARGETS_TO_BUILD ${LLVM_EXPERIMENTAL_TARGETS_TO_BUILD}) list(REMOVE_DUPLICATES LLVM_TARGETS_TO_BUILD) -option(LLVM_ENABLE_PIC "Build Position-Independent Code" ON) +if (NOT CMAKE_SYSTEM_NAME MATCHES "OS390") + option(LLVM_ENABLE_PIC "Build Position-Independent Code" ON) +endif() option(LLVM_ENABLE_MODULES "Compile with C++ modules enabled." OFF) if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") option(LLVM_ENABLE_MODULE_DEBUGGING "Compile with -gmodules." ON) @@ -1127,6 +1129,8 @@ if (CMAKE_SYSTEM_NAME MATCHES "OS390") add_compile_definitions(_XOPEN_SOURCE=600) add_compile_definitions(_OPEN_SYS) # Needed for process information. add_compile_definitions(_OPEN_SYS_FILE_EXT) # Needed for EBCDIC I/O. + add_compile_definitions(_EXT) # Needed for file data. + add_compile_definitions(_UNIX03_THREADS) # Multithreading support. endif() # Build with _FILE_OFFSET_BITS=64 on Solaris to match g++ >= 9. diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake index 97fd908d08f176b20de971125e5a244228887ae7..0699a8586fcc7e5105f831fbbf641f47b4538372 100644 --- a/llvm/cmake/modules/HandleLLVMOptions.cmake +++ b/llvm/cmake/modules/HandleLLVMOptions.cmake @@ -988,6 +988,9 @@ if(LLVM_USE_SANITIZER) endif() # Prepare ASAN runtime if needed if (LLVM_USE_SANITIZER MATCHES ".*Address.*") + # lld string tail merging interacts badly with ASAN on Windows, turn it off here + # See https://github.com/llvm/llvm-project/issues/62078 + append("/opt:nolldtailmerge" CMAKE_EXE_LINKER_FLAGS CMAKE_MODULE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS) if (${CMAKE_MSVC_RUNTIME_LIBRARY} MATCHES "^(MultiThreaded|MultiThreadedDebug)$") append("/wholearchive:clang_rt.asan-${arch}.lib /wholearchive:clang_rt.asan_cxx-${arch}.lib" CMAKE_EXE_LINKER_FLAGS) diff --git a/llvm/docs/CoverageMappingFormat.rst b/llvm/docs/CoverageMappingFormat.rst index cbabad7a332cbc4c91b9149a22920f8d2a4f67f1..c5cd1f795aab1b7d43fc0f0c44b5527c8be78ccc 100644 --- a/llvm/docs/CoverageMappingFormat.rst +++ b/llvm/docs/CoverageMappingFormat.rst @@ -178,7 +178,7 @@ defined inside macros, like this example demonstrates: Counter: ^^^^^^^^ -A coverage mapping counter can represents a reference to the profile +A coverage mapping counter can represent a reference to the profile instrumentation counter. The execution count for a region with such counter is determined by looking up the value of the corresponding profile instrumentation counter. diff --git a/llvm/docs/GettingStarted.rst b/llvm/docs/GettingStarted.rst index 24ed7118ee814ada8b646e8a532bc7fec1b6b5c4..da9cc8aea6d32d8aed7754a1a467fde741e0bee7 100644 --- a/llvm/docs/GettingStarted.rst +++ b/llvm/docs/GettingStarted.rst @@ -43,7 +43,7 @@ Getting the Source Code and Building LLVM * You are likely only interested in the main branch moving forward, if you don't want `git fetch` (or `git pull`) to download user branches, use: - ``sed 's#fetch = +refs/heads/\*:refs/remotes/origin/\*#fetch = +refs/heads/main:refs/remotes/origin/main# -i llvm-project/.git/config`` + ``sed 's#fetch = +refs/heads/\*:refs/remotes/origin/\*#fetch = +refs/heads/main:refs/remotes/origin/main#' -i llvm-project/.git/config`` #. Configure and build LLVM and Clang: diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index f5e8065ca1dc6054ea2e5101af486e1137eac081..adda52b33c789b98712c33bdd9db96aea08e0fb3 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -4299,6 +4299,11 @@ constants and smaller complex constants. "``< i32 42, i32 11, i32 74, i32 100 >``". Vector constants must have :ref:`vector type `, and the number and types of elements must match those specified by the type. + + When creating a vector whose elements have the same constant value, the + preferred syntax is ``splat ( Val)``. For example: "``splat (i32 11)``". + These vector constants must have ::ref:`vector type ` with an + element type that matches the ``splat`` operand. **Zero initialization** The string '``zeroinitializer``' can be used to zero initialize a value to zero of *any* type, including scalar and diff --git a/llvm/docs/MIRLangRef.rst b/llvm/docs/MIRLangRef.rst index 223701599ebb8b5f670f299a9a9196d14d9f7643..52ff24daa7fbabb0cd6ddc7e988688d7cb6589e6 100644 --- a/llvm/docs/MIRLangRef.rst +++ b/llvm/docs/MIRLangRef.rst @@ -58,12 +58,12 @@ for the post register allocation pseudo instruction expansion pass, you can specify the machine copy propagation pass in the ``-stop-after`` option, as it runs just before the pass that we are trying to test: - ``llc -stop-after=machine-cp bug-trigger.ll > test.mir`` + ``llc -stop-after=machine-cp bug-trigger.ll -o test.mir`` If the same pass is run multiple times, a run index can be included after the name with a comma. - ``llc -stop-after=dead-mi-elimination,1 bug-trigger.ll > test.mir`` + ``llc -stop-after=dead-mi-elimination,1 bug-trigger.ll -o test.mir`` After generating the input MIR file, you'll have to add a run line that uses the ``-run-pass`` option to it. In order to test the post register allocation diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst index f58ae03a6efcf08b24e9c75b93fd00d78e735020..d5c634d2f29af5726a557d2a0b56dbdab8df2d35 100644 --- a/llvm/docs/ReleaseNotes.rst +++ b/llvm/docs/ReleaseNotes.rst @@ -92,6 +92,15 @@ Changes to Interprocedural Optimizations Changes to the AArch64 Backend ------------------------------ +* Added support for Cortex-A520, Cortex-A720 and Cortex-X4 CPUs. + +* Neoverse-N2 was incorrectly marked as an Armv8.5a core. This has been + changed to an Armv9.0a core. However, crypto options are not enabled + by default for Armv9 cores, so `-mcpu=neoverse-n2+crypto` is now required + to enable crypto for this core. As far as the compiler is concerned, + Armv9.0a has the same features enabled as Armv8.5a, with the exception + of crypto. + Changes to the AMDGPU Backend ----------------------------- @@ -102,11 +111,11 @@ Changes to the AMDGPU Backend * Implemented :ref:`llvm.get.rounding ` -* Added support for Cortex-A520, Cortex-A720 and Cortex-X4 CPUs. - Changes to the ARM Backend -------------------------- +* Added support for Cortex-M52 CPUs. + Changes to the AVR Backend -------------------------- @@ -204,6 +213,19 @@ Changes to the C API on zext instructions, and ``LLVMGetIsDisjoint`` and ``LLVMSetIsDisjoint`` for getting/setting the new disjoint flag on or instructions. +* Added the following functions for manipulating operand bundles, as well as + building ``call`` and ``invoke`` instructions that use operand bundles: + + * ``LLVMBuildCallWithOperandBundles`` + * ``LLVMBuildInvokeWithOperandBundles`` + * ``LLVMCreateOperandBundle`` + * ``LLVMDisposeOperandBundle`` + * ``LLVMGetNumOperandBundles`` + * ``LLVMGetOperandBundleAtIndex`` + * ``LLVMGetNumOperandBundleArgs`` + * ``LLVMGetOperandBundleArgAtIndex`` + * ``LLVMGetOperandBundleTag`` + Changes to the CodeGen infrastructure ------------------------------------- diff --git a/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/LLJITWithRemoteDebugging.cpp b/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/LLJITWithRemoteDebugging.cpp index 291f14e1d7d7aea368dc4e860071e6d36f83811e..9001125060583f5bd868d5ab647e58ac8d5a19a5 100644 --- a/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/LLJITWithRemoteDebugging.cpp +++ b/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/LLJITWithRemoteDebugging.cpp @@ -88,6 +88,7 @@ #include "llvm/Support/InitLLVM.h" #include "llvm/Support/TargetSelect.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/TargetParser/Host.h" #include "../ExampleModules.h" #include "RemoteJITUtils.h" @@ -173,24 +174,15 @@ int main(int argc, char *argv[]) { TSMs.push_back(ExitOnErr(parseExampleModuleFromFile(Path))); } - StringRef TT; + std::string TT; StringRef MainModuleName; TSMs.front().withModuleDo([&MainModuleName, &TT](Module &M) { MainModuleName = M.getName(); TT = M.getTargetTriple(); + if (TT.empty()) + TT = sys::getProcessTriple(); }); - for (const ThreadSafeModule &TSM : TSMs) - ExitOnErr(TSM.withModuleDo([TT, MainModuleName](Module &M) -> Error { - if (M.getTargetTriple() != TT) - return make_error( - formatv("Different target triples in input files:\n" - " '{0}' in '{1}'\n '{2}' in '{3}'", - TT, MainModuleName, M.getTargetTriple(), M.getName()), - inconvertibleErrorCode()); - return Error::success(); - })); - // Create a target machine that matches the input triple. JITTargetMachineBuilder JTMB((Triple(TT))); JTMB.setCodeModel(CodeModel::Small); diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h index a575ec3709fe4bc90ec060004812d50e41c08e61..7cb809d378c957487b7bf10c27b4a8fb3e94267b 100644 --- a/llvm/include/llvm-c/Core.h +++ b/llvm/include/llvm-c/Core.h @@ -3000,6 +3000,74 @@ LLVMValueRef LLVMMDNodeInContext(LLVMContextRef C, LLVMValueRef *Vals, /** Deprecated: Use LLVMMDNodeInContext2 instead. */ LLVMValueRef LLVMMDNode(LLVMValueRef *Vals, unsigned Count); +/** + * @} + */ + +/** + * @defgroup LLVMCCoreOperandBundle Operand Bundles + * + * Functions in this group operate on LLVMOperandBundleRef instances that + * correspond to llvm::OperandBundleDef instances. + * + * @see llvm::OperandBundleDef + * + * @{ + */ + +/** + * Create a new operand bundle. + * + * Every invocation should be paired with LLVMDisposeOperandBundle() or memory + * will be leaked. + * + * @param Tag Tag name of the operand bundle + * @param TagLen Length of Tag + * @param Args Memory address of an array of bundle operands + * @param NumArgs Length of Args + */ +LLVMOperandBundleRef LLVMCreateOperandBundle(const char *Tag, size_t TagLen, + LLVMValueRef *Args, + unsigned NumArgs); + +/** + * Destroy an operand bundle. + * + * This must be called for every created operand bundle or memory will be + * leaked. + */ +void LLVMDisposeOperandBundle(LLVMOperandBundleRef Bundle); + +/** + * Obtain the tag of an operand bundle as a string. + * + * @param Bundle Operand bundle to obtain tag of. + * @param Len Out parameter which holds the length of the returned string. + * @return The tag name of Bundle. + * @see OperandBundleDef::getTag() + */ +const char *LLVMGetOperandBundleTag(LLVMOperandBundleRef Bundle, size_t *Len); + +/** + * Obtain the number of operands for an operand bundle. + * + * @param Bundle Operand bundle to obtain operand count of. + * @return The number of operands. + * @see OperandBundleDef::input_size() + */ +unsigned LLVMGetNumOperandBundleArgs(LLVMOperandBundleRef Bundle); + +/** + * Obtain the operand for an operand bundle at the given index. + * + * @param Bundle Operand bundle to obtain operand of. + * @param Index An operand index, must be less than + * LLVMGetNumOperandBundleArgs(). + * @return The operand. + */ +LLVMValueRef LLVMGetOperandBundleArgAtIndex(LLVMOperandBundleRef Bundle, + unsigned Index); + /** * @} */ @@ -3451,6 +3519,24 @@ LLVMTypeRef LLVMGetCalledFunctionType(LLVMValueRef C); */ LLVMValueRef LLVMGetCalledValue(LLVMValueRef Instr); +/** + * Obtain the number of operand bundles attached to this instruction. + * + * This only works on llvm::CallInst and llvm::InvokeInst instructions. + * + * @see llvm::CallBase::getNumOperandBundles() + */ +unsigned LLVMGetNumOperandBundles(LLVMValueRef C); + +/** + * Obtain the operand bundle attached to this instruction at the given index. + * Use LLVMDisposeOperandBundle to free the operand bundle. + * + * This only works on llvm::CallInst and llvm::InvokeInst instructions. + */ +LLVMOperandBundleRef LLVMGetOperandBundleAtIndex(LLVMValueRef C, + unsigned Index); + /** * Obtain whether a call instruction is a tail call. * @@ -3815,6 +3901,10 @@ LLVMValueRef LLVMBuildInvoke2(LLVMBuilderRef, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, const char *Name); +LLVMValueRef LLVMBuildInvokeWithOperandBundles( + LLVMBuilderRef, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMValueRef *Args, + unsigned NumArgs, LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, + LLVMOperandBundleRef *Bundles, unsigned NumBundles, const char *Name); LLVMValueRef LLVMBuildUnreachable(LLVMBuilderRef); /* Exception Handling */ @@ -4121,6 +4211,11 @@ LLVMValueRef LLVMBuildPhi(LLVMBuilderRef, LLVMTypeRef Ty, const char *Name); LLVMValueRef LLVMBuildCall2(LLVMBuilderRef, LLVMTypeRef, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, const char *Name); +LLVMValueRef +LLVMBuildCallWithOperandBundles(LLVMBuilderRef, LLVMTypeRef, LLVMValueRef Fn, + LLVMValueRef *Args, unsigned NumArgs, + LLVMOperandBundleRef *Bundles, + unsigned NumBundles, const char *Name); LLVMValueRef LLVMBuildSelect(LLVMBuilderRef, LLVMValueRef If, LLVMValueRef Then, LLVMValueRef Else, const char *Name); diff --git a/llvm/include/llvm-c/LLJIT.h b/llvm/include/llvm-c/LLJIT.h index a06133aac4fb0630d417fbf3c0fb74b54862889d..a58c3b8bbef700d5fc3d348184327193dc8afe25 100644 --- a/llvm/include/llvm-c/LLJIT.h +++ b/llvm/include/llvm-c/LLJIT.h @@ -1,4 +1,4 @@ -/*===----------- llvm-c/LLJIT.h - OrcV2 LLJIT C bindings --------*- C++ -*-===*\ +/*===----------- llvm-c/LLJIT.h - OrcV2 LLJIT C bindings ----------*- C -*-===*\ |* *| |* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| |* Exceptions. *| diff --git a/llvm/include/llvm-c/LLJITUtils.h b/llvm/include/llvm-c/LLJITUtils.h new file mode 100644 index 0000000000000000000000000000000000000000..940097432b78f53c8fd86e25146ec20659a02815 --- /dev/null +++ b/llvm/include/llvm-c/LLJITUtils.h @@ -0,0 +1,52 @@ +/*===------- llvm-c/LLJITUtils.h - Advanced LLJIT features --------*- C -*-===*\ +|* *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This header declares the C interface for extra utilities to be used with *| +|* the LLJIT class from the llvm-c/LLJIT.h header. It requires to following *| +|* link libraries in addition to libLLVMOrcJIT.a: *| +|* - libLLVMOrcDebugging.a *| +|* *| +|* Many exotic languages can interoperate with C code but have a harder time *| +|* with C++ due to name mangling. So in addition to C, this interface enables *| +|* tools written in such languages. *| +|* *| +|* Note: This interface is experimental. It is *NOT* stable, and may be *| +|* changed without warning. Only C API usage documentation is *| +|* provided. See the C++ documentation for all higher level ORC API *| +|* details. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_C_LLJITUTILS_H +#define LLVM_C_LLJITUTILS_H + +#include "llvm-c/LLJIT.h" + +LLVM_C_EXTERN_C_BEGIN + +/** + * @defgroup LLVMCExecutionEngineLLJITUtils LLJIT Utilities + * @ingroup LLVMCExecutionEngineLLJIT + * + * @{ + */ + +/** + * Install the plugin that submits debug objects to the executor. Executors must + * expose the llvm_orc_registerJITLoaderGDBWrapper symbol. + */ +LLVMErrorRef LLVMOrcLLJITEnableDebugSupport(LLVMOrcLLJITRef J); + +/** + * @} + */ + +LLVM_C_EXTERN_C_END + +#endif /* LLVM_C_LLJITUTILS_H */ diff --git a/llvm/include/llvm-c/Types.h b/llvm/include/llvm-c/Types.h index 4e9967372d79f5dbfa5e83a0e3395e5784e484f7..d5474d986309fa02fc6b855628226f441af143e8 100644 --- a/llvm/include/llvm-c/Types.h +++ b/llvm/include/llvm-c/Types.h @@ -132,6 +132,11 @@ typedef struct LLVMOpaquePassManager *LLVMPassManagerRef; * @see llvm::Use */ typedef struct LLVMOpaqueUse *LLVMUseRef; +/** + * @see llvm::OperandBundleDef + */ +typedef struct LLVMOpaqueOperandBundle *LLVMOperandBundleRef; + /** * Used to represent an attributes. * diff --git a/llvm/include/llvm/ADT/SmallString.h b/llvm/include/llvm/ADT/SmallString.h index 0052c86fb37b82dcdf577a7acf06e3a47f54da61..02fa28fc856d7032f6c1fca1560da20694016c46 100644 --- a/llvm/include/llvm/ADT/SmallString.h +++ b/llvm/include/llvm/ADT/SmallString.h @@ -120,15 +120,11 @@ public: /// @name String Predicates /// @{ - /// startswith - Check if this string starts with the given \p Prefix. - bool startswith(StringRef Prefix) const { - return str().startswith(Prefix); - } + /// starts_with - Check if this string starts with the given \p Prefix. + bool starts_with(StringRef Prefix) const { return str().starts_with(Prefix); } - /// endswith - Check if this string ends with the given \p Suffix. - bool endswith(StringRef Suffix) const { - return str().endswith(Suffix); - } + /// ends_with - Check if this string ends with the given \p Suffix. + bool ends_with(StringRef Suffix) const { return str().ends_with(Suffix); } /// @} /// @name String Searching diff --git a/llvm/include/llvm/ADT/SparseBitVector.h b/llvm/include/llvm/ADT/SparseBitVector.h index 1e00c1386187510a7d6a63aece60bc0290c18fdf..7151af6146e6e40c2901ac8ea79015903eeb7c54 100644 --- a/llvm/include/llvm/ADT/SparseBitVector.h +++ b/llvm/include/llvm/ADT/SparseBitVector.h @@ -15,8 +15,8 @@ #ifndef LLVM_ADT_SPARSEBITVECTOR_H #define LLVM_ADT_SPARSEBITVECTOR_H +#include "llvm/ADT/bit.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include #include diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h index 235a7b27c384406cb6a2d02557658ef96f707e49..4e69d5b633546de094e484d53ad15eaaba06888c 100644 --- a/llvm/include/llvm/ADT/StringRef.h +++ b/llvm/include/llvm/ADT/StringRef.h @@ -264,12 +264,6 @@ namespace llvm { /// Check if this string starts with the given \p Prefix, ignoring case. [[nodiscard]] bool starts_with_insensitive(StringRef Prefix) const; - [[nodiscard]] LLVM_DEPRECATED( - "Use starts_with_insensitive instead", - "starts_with_insensitive") bool startswith_insensitive(StringRef Prefix) - const { - return starts_with_insensitive(Prefix); - } /// Check if this string ends with the given \p Suffix. [[nodiscard]] bool ends_with(StringRef Suffix) const { @@ -283,12 +277,6 @@ namespace llvm { /// Check if this string ends with the given \p Suffix, ignoring case. [[nodiscard]] bool ends_with_insensitive(StringRef Suffix) const; - [[nodiscard]] LLVM_DEPRECATED( - "Use ends_with_insensitive instead", - "ends_with_insensitive") bool endswith_insensitive(StringRef Suffix) - const { - return ends_with_insensitive(Suffix); - } /// @} /// @name String Searching diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h index 91848a91c17e6e67e69f8b2b39c57736225a31e6..fd884f2a2f55b025f98b0e615d5ce271016597e8 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h @@ -16,7 +16,6 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/IR/Constants.h" #include "llvm/IR/ValueHandle.h" diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index 8635bdd470ee69f56324b80475495cdebafcc3cd..fb6f3287e3d262b59fc32ad4b93f1b8a5ddcb51c 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -2376,12 +2376,12 @@ public: bool IsZeroCmp) const override { return Impl.enableMemCmpExpansion(OptSize, IsZeroCmp); } - bool enableInterleavedAccessVectorization() override { - return Impl.enableInterleavedAccessVectorization(); - } bool enableSelectOptimize() override { return Impl.enableSelectOptimize(); } + bool enableInterleavedAccessVectorization() override { + return Impl.enableInterleavedAccessVectorization(); + } bool enableMaskedInterleavedAccessVectorization() override { return Impl.enableMaskedInterleavedAccessVectorization(); } diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h index 793287c772b55e8af89f815b7e30bb34c1bb0031..54bc3e582e01aecd9e07691c8d8d8323c32734a0 100644 --- a/llvm/include/llvm/AsmParser/LLParser.h +++ b/llvm/include/llvm/AsmParser/LLParser.h @@ -51,15 +51,23 @@ namespace llvm { /// or a symbolic (%var) reference. This is just a discriminated union. struct ValID { enum { - t_LocalID, t_GlobalID, // ID in UIntVal. - t_LocalName, t_GlobalName, // Name in StrVal. - t_APSInt, t_APFloat, // Value in APSIntVal/APFloatVal. - t_Null, t_Undef, t_Zero, t_None, t_Poison, // No value. - t_EmptyArray, // No value: [] - t_Constant, // Value in ConstantVal. - t_InlineAsm, // Value in FTy/StrVal/StrVal2/UIntVal. - t_ConstantStruct, // Value in ConstantStructElts. - t_PackedConstantStruct // Value in ConstantStructElts. + t_LocalID, // ID in UIntVal. + t_GlobalID, // ID in UIntVal. + t_LocalName, // Name in StrVal. + t_GlobalName, // Name in StrVal. + t_APSInt, // Value in APSIntVal. + t_APFloat, // Value in APFloatVal. + t_Null, // No value. + t_Undef, // No value. + t_Zero, // No value. + t_None, // No value. + t_Poison, // No value. + t_EmptyArray, // No value: [] + t_Constant, // Value in ConstantVal. + t_ConstantSplat, // Value in ConstantVal. + t_InlineAsm, // Value in FTy/StrVal/StrVal2/UIntVal. + t_ConstantStruct, // Value in ConstantStructElts. + t_PackedConstantStruct // Value in ConstantStructElts. } Kind = t_LocalID; LLLexer::LocTy Loc; @@ -407,9 +415,10 @@ namespace llvm { std::map, WholeProgramDevirtResolution::ByArg> &ResByArg); bool parseArgs(std::vector &Args); - void addGlobalValueToIndex(std::string Name, GlobalValue::GUID, + bool addGlobalValueToIndex(std::string Name, GlobalValue::GUID, GlobalValue::LinkageTypes Linkage, unsigned ID, - std::unique_ptr Summary); + std::unique_ptr Summary, + LocTy Loc); bool parseOptionalAllocs(std::vector &Allocs); bool parseMemProfs(std::vector &MIBs); bool parseAllocType(uint8_t &AllocType); diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h index 0aa0093e8efbd3e4c77fd871c89b7f8dfbe8f040..147cf56c821aa1fdfbaada0ad2a78a09a5c08924 100644 --- a/llvm/include/llvm/AsmParser/LLToken.h +++ b/llvm/include/llvm/AsmParser/LLToken.h @@ -336,6 +336,7 @@ enum Kind { kw_extractelement, kw_insertelement, kw_shufflevector, + kw_splat, kw_extractvalue, kw_insertvalue, kw_blockaddress, diff --git a/llvm/include/llvm/BinaryFormat/DynamicTags.def b/llvm/include/llvm/BinaryFormat/DynamicTags.def index f393b82406b41d9018558472068edb054438cf3e..1502d375f5c45da8b63116c68edfc552cef60bc0 100644 --- a/llvm/include/llvm/BinaryFormat/DynamicTags.def +++ b/llvm/include/llvm/BinaryFormat/DynamicTags.def @@ -132,6 +132,12 @@ AARCH64_DYNAMIC_TAG(AARCH64_MEMTAG_STACK, 0x7000000c) AARCH64_DYNAMIC_TAG(AARCH64_MEMTAG_GLOBALS, 0x7000000d) AARCH64_DYNAMIC_TAG(AARCH64_MEMTAG_GLOBALSSZ, 0x7000000f) +// AArch64 specific dynamic table entries for RELR auth relocations as described here: +// https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#dynamic-section +AARCH64_DYNAMIC_TAG(AARCH64_AUTH_RELRSZ, 0x70000011) +AARCH64_DYNAMIC_TAG(AARCH64_AUTH_RELR, 0x70000012) +AARCH64_DYNAMIC_TAG(AARCH64_AUTH_RELRENT, 0x70000013) + // Hexagon specific dynamic table entries HEXAGON_DYNAMIC_TAG(HEXAGON_SYMSZ, 0x70000000) HEXAGON_DYNAMIC_TAG(HEXAGON_VER, 0x70000001) diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index d1ce8e20b4be73a4f8a99105718f7571cf288b5b..40c795410f95adac858e6ecd4cd30a1a9482dad7 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -1060,6 +1060,9 @@ enum : unsigned { SHT_ARM_ATTRIBUTES = 0x70000003U, SHT_ARM_DEBUGOVERLAY = 0x70000004U, SHT_ARM_OVERLAYSECTION = 0x70000005U, + // Special aarch64-specific section for MTE support, as described in: + // https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#section-types + SHT_AARCH64_AUTH_RELR = 0x70000004U, // Special aarch64-specific sections for MTE support, as described in: // https://github.com/ARM-software/abi-aa/blob/main/memtagabielf64/memtagabielf64.rst#7section-types SHT_AARCH64_MEMTAG_GLOBALS_STATIC = 0x70000007U, @@ -1647,6 +1650,11 @@ enum { NT_ANDROID_TYPE_MEMTAG = 4, }; +// ARM note types. +enum { + NT_ARM_TYPE_PAUTH_ABI_TAG = 1, +}; + // Memory tagging values used in NT_ANDROID_TYPE_MEMTAG notes. enum { // Enumeration to determine the tagging mode. In Android-land, 'SYNC' means diff --git a/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def b/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def index b507109b19e1b90dac6b055b181df303da54eacd..30375de420e30244abc21dcbb58a1ced90e03a71 100644 --- a/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def +++ b/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def @@ -121,6 +121,7 @@ ELF_RELOC(R_AARCH64_TLSLE_LDST128_TPREL_LO12, 0x23a) ELF_RELOC(R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC, 0x23b) ELF_RELOC(R_AARCH64_TLSLD_LDST128_DTPREL_LO12, 0x23c) ELF_RELOC(R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC, 0x23d) +ELF_RELOC(R_AARCH64_AUTH_ABS64, 0x244) // Dynamic relocations start ELF_RELOC(R_AARCH64_COPY, 0x400) ELF_RELOC(R_AARCH64_GLOB_DAT, 0x401) @@ -134,7 +135,7 @@ ELF_RELOC(R_AARCH64_TLS_DTPREL64, 0x405) ELF_RELOC(R_AARCH64_TLS_TPREL64, 0x406) ELF_RELOC(R_AARCH64_TLSDESC, 0x407) ELF_RELOC(R_AARCH64_IRELATIVE, 0x408) -ELF_RELOC(R_AARCH64_AUTH_ABS64, 0xe100) +ELF_RELOC(R_AARCH64_AUTH_RELATIVE, 0x411) // ELF_RELOC(R_AARCH64_P32_NONE, 0) ELF_RELOC(R_AARCH64_P32_ABS32, 0x001) diff --git a/llvm/include/llvm/CodeGen/CallingConvLower.h b/llvm/include/llvm/CodeGen/CallingConvLower.h index 0989fae54b3ac6fd46f36c35fb96b00addba0ad4..932a2a94ab1f1a94784d67d31f2ead1ebd594018 100644 --- a/llvm/include/llvm/CodeGen/CallingConvLower.h +++ b/llvm/include/llvm/CodeGen/CallingConvLower.h @@ -14,6 +14,7 @@ #ifndef LLVM_CODEGEN_CALLINGCONVLOWER_H #define LLVM_CODEGEN_CALLINGCONVLOWER_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/Register.h" #include "llvm/CodeGen/TargetCallingConv.h" diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h index 076719abd0356b0287c2e9d736236717457ca3be..bb139ef2eb351007e375ce3e5218d3fe797d9f68 100644 --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -23,7 +23,9 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/CodeGen/CallBrPrepare.h" +#include "llvm/CodeGen/DwarfEHPrepare.h" #include "llvm/CodeGen/ExpandReductions.h" +#include "llvm/CodeGen/InterleavedAccess.h" #include "llvm/CodeGen/MachinePassManager.h" #include "llvm/CodeGen/PreISelIntrinsicLowering.h" #include "llvm/CodeGen/ReplaceWithVeclib.h" @@ -677,14 +679,14 @@ void CodeGenPassBuilder::addPassesToHandleExceptions( case ExceptionHandling::DwarfCFI: case ExceptionHandling::ARM: case ExceptionHandling::AIX: - addPass(DwarfEHPass(getOptLevel())); + addPass(DwarfEHPreparePass(&TM)); break; case ExceptionHandling::WinEH: // We support using both GCC-style and MSVC-style exceptions on Windows, so // add both preparation passes. Each pass will only actually run if it // recognizes the personality function. addPass(WinEHPreparePass()); - addPass(DwarfEHPass(getOptLevel())); + addPass(DwarfEHPreparePass(&TM)); break; case ExceptionHandling::Wasm: // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h index 0b167ce9650d07016a8f3ba1f1559989a656844c..e423e48fd31ea35fa4c75f09c8158a467a2ffc9e 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h @@ -19,6 +19,7 @@ #include "llvm/CodeGen/LowLevelType.h" #include "llvm/CodeGen/TargetOpcodes.h" #include +#include namespace llvm { struct LegalityQuery; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index 365d2223a81c1d0ffef2c5294a209477daf76321..711ba10247c34d92a7235f4a222b570ecdf8e231 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -432,6 +432,7 @@ public: LegalizeResult lowerVectorReduction(MachineInstr &MI); LegalizeResult lowerMemcpyInline(MachineInstr &MI); LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen = 0); + LegalizeResult lowerVAArg(MachineInstr &MI); }; /// Helper function that creates a libcall to the given \p Name using the given diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index e0101a5ac1ca80498397e3452fda4ea2683f4ecd..1387a0a37561c4bf04486a3caf64a9abe6bfada3 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -1194,7 +1194,7 @@ public: const SrcOp &Op0, const SrcOp &Op1, std::optional Flags = std::nullopt); - /// Build and insert a \p Res = G_IS_FPCLASS \p Pred, \p Src, \p Mask + /// Build and insert a \p Res = G_IS_FPCLASS \p Src, \p Mask MachineInstrBuilder buildIsFPClass(const DstOp &Res, const SrcOp &Src, unsigned Mask) { return buildInstr(TargetOpcode::G_IS_FPCLASS, {Res}, @@ -1529,6 +1529,11 @@ public: /// Build and insert `G_FENCE Ordering, Scope`. MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope); + /// Build and insert G_PREFETCH \p Addr, \p RW, \p Locality, \p CacheType + MachineInstrBuilder buildPrefetch(const SrcOp &Addr, unsigned RW, + unsigned Locality, unsigned CacheType, + MachineMemOperand &MMO); + /// Build and insert \p Dst = G_FREEZE \p Src MachineInstrBuilder buildFreeze(const DstOp &Dst, const SrcOp &Src) { return buildInstr(TargetOpcode::G_FREEZE, {Dst}, {Src}); diff --git a/llvm/include/llvm/CodeGen/InterleavedAccess.h b/llvm/include/llvm/CodeGen/InterleavedAccess.h new file mode 100644 index 0000000000000000000000000000000000000000..31bd19a3191a273e7baef4d34f38100e4b4bef39 --- /dev/null +++ b/llvm/include/llvm/CodeGen/InterleavedAccess.h @@ -0,0 +1,34 @@ +//===---- llvm/CodeGen/InterleavedAccess.h ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the declaration of the InterleavedAccessPass class, +/// its corresponding pass name is `interleaved-access`. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_INTERLEAVEDACCESS_H +#define LLVM_CODEGEN_INTERLEAVEDACCESS_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class TargetMachine; + +class InterleavedAccessPass : public PassInfoMixin { + const TargetMachine *TM; + +public: + explicit InterleavedAccessPass(const TargetMachine *TM) : TM(TM) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); +}; + +} // namespace llvm + +#endif // LLVM_CODEGEN_INTERLEAVEDACCESS_H diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def index 1e9e5838841b29696295d39db8b7eb4b5aa8cad1..e6e979a4582c7a7908def867219234d0d81d3bfd 100644 --- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def +++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def @@ -38,11 +38,13 @@ FUNCTION_ANALYSIS("targetir", TargetIRAnalysis, #endif FUNCTION_PASS("callbrprepare", CallBrPreparePass, ()) FUNCTION_PASS("consthoist", ConstantHoistingPass, ()) +FUNCTION_PASS("dwarf-eh-prepare", DwarfEHPreparePass, (TM)) FUNCTION_PASS("ee-instrument", EntryExitInstrumenterPass, (false)) FUNCTION_PASS("expand-large-div-rem", ExpandLargeDivRemPass, ()) FUNCTION_PASS("expand-large-fp-convert", ExpandLargeFpConvertPass, ()) FUNCTION_PASS("expand-reductions", ExpandReductionsPass, ()) FUNCTION_PASS("expandvp", ExpandVectorPredicationPass, ()) +FUNCTION_PASS("interleaved-access", InterleavedAccessPass, (TM)) FUNCTION_PASS("lower-constant-intrinsics", LowerConstantIntrinsicsPass, ()) FUNCTION_PASS("lowerinvoke", LowerInvokePass, ()) FUNCTION_PASS("mergeicmps", MergeICmpsPass, ()) @@ -122,12 +124,10 @@ DUMMY_FUNCTION_PASS("atomic-expand", AtomicExpandPass, ()) DUMMY_FUNCTION_PASS("cfguard-check", CFGuardCheckPass, ()) DUMMY_FUNCTION_PASS("cfguard-dispatch", CFGuardDispatchPass, ()) DUMMY_FUNCTION_PASS("codegenprepare", CodeGenPreparePass, ()) -DUMMY_FUNCTION_PASS("dwarfehprepare", DwarfEHPass, ()) DUMMY_FUNCTION_PASS("expandmemcmp", ExpandMemCmpPass, ()) DUMMY_FUNCTION_PASS("gc-info-printer", GCInfoPrinterPass, ()) DUMMY_FUNCTION_PASS("gc-lowering", GCLoweringPass, ()) DUMMY_FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass, ()) -DUMMY_FUNCTION_PASS("interleaved-access", InterleavedAccessPass, ()) DUMMY_FUNCTION_PASS("select-optimize", SelectOptimizePass, ()) DUMMY_FUNCTION_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ()) DUMMY_FUNCTION_PASS("sjljehprepare", SjLjEHPreparePass, ()) diff --git a/llvm/include/llvm/CodeGen/SchedulerRegistry.h b/llvm/include/llvm/CodeGen/SchedulerRegistry.h index 0c356e62ae4eeab563cce882a210ae134462e4b6..cf648d1316c98c2e0723564d4b6e4185c67b0cdf 100644 --- a/llvm/include/llvm/CodeGen/SchedulerRegistry.h +++ b/llvm/include/llvm/CodeGen/SchedulerRegistry.h @@ -63,7 +63,7 @@ public: ScheduleDAGSDNodes *createBURRListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel); -/// createBURRListDAGScheduler - This creates a bottom up list scheduler that +/// createSourceListDAGScheduler - This creates a bottom up list scheduler that /// schedules nodes in source code order when possible. ScheduleDAGSDNodes *createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel); diff --git a/llvm/include/llvm/CodeGen/TargetSchedule.h b/llvm/include/llvm/CodeGen/TargetSchedule.h index 3d39798790cdceb97505b59dad58e7e57fd65bc8..bfe4234abf8eb773e521b36cb9a8b3dcecf2aa02 100644 --- a/llvm/include/llvm/CodeGen/TargetSchedule.h +++ b/llvm/include/llvm/CodeGen/TargetSchedule.h @@ -46,7 +46,7 @@ class TargetSchedModel { unsigned computeInstrLatency(const MCSchedClassDesc &SCDesc) const; public: - TargetSchedModel() : SchedModel(MCSchedModel::GetDefaultSchedModel()) {} + TargetSchedModel() : SchedModel(MCSchedModel::Default) {} /// Initialize the machine model for instruction scheduling. /// diff --git a/llvm/include/llvm/DebugInfo/CodeView/CodeView.h b/llvm/include/llvm/DebugInfo/CodeView/CodeView.h index 62e559e2cebaef7b1590b96d834ef4f97decf866..0bfd06e05bd89a59dc8fc11658f7303c76f545e0 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/CodeView.h +++ b/llvm/include/llvm/DebugInfo/CodeView/CodeView.h @@ -16,6 +16,7 @@ #include #include +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/Support/Endian.h" namespace llvm { @@ -51,15 +52,15 @@ enum SymbolKind : uint16_t { #define CV_DEFINE_ENUM_CLASS_FLAGS_OPERATORS(Class) \ inline Class operator|(Class a, Class b) { \ - return static_cast(static_cast>(a) | \ - static_cast>(b)); \ + return static_cast(llvm::to_underlying(a) | \ + llvm::to_underlying(b)); \ } \ inline Class operator&(Class a, Class b) { \ - return static_cast(static_cast>(a) & \ - static_cast>(b)); \ + return static_cast(llvm::to_underlying(a) & \ + llvm::to_underlying(b)); \ } \ inline Class operator~(Class a) { \ - return static_cast(~static_cast>(a)); \ + return static_cast(~llvm::to_underlying(a)); \ } \ inline Class &operator|=(Class &a, Class b) { \ a = a | b; \ diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h b/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h index 01de8b49dd78fd49f800aa954c7f08a046c96636..0adbd25cb369ea314bb0410cfb464020ced7fc90 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h @@ -10,6 +10,7 @@ #define LLVM_DEBUGINFO_PDB_NATIVE_FORMATUTIL_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/ADT/StringRef.h" #include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/Support/Endian.h" @@ -34,8 +35,7 @@ namespace pdb { return Ret; template std::string formatUnknownEnum(T Value) { - return formatv("unknown ({0})", static_cast>(Value)) - .str(); + return formatv("unknown ({0})", llvm::to_underlying(Value)).str(); } std::string formatSegmentOffset(uint16_t Segment, uint32_t Offset); diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h index 27a90ebef3d6d6a989ef5750b24f74c479f9f871..40b9339eb5316d8f27de9353a2c1cc0293343cf3 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h @@ -646,7 +646,7 @@ inline Symbol &createAnonymousPointer(LinkGraph &G, Section &PointerSection, inline Block &createPointerJumpStubBlock(LinkGraph &G, Section &StubSection, Symbol &PointerSymbol) { auto &B = G.createContentBlock(StubSection, PointerJumpStubContent, - orc::ExecutorAddr(~uint64_t(11)), 1, 0); + orc::ExecutorAddr(~uint64_t(11)), 4, 0); B.addEdge(Page21, 0, PointerSymbol, 0); B.addEdge(PageOffset12, 4, PointerSymbol, 0); return B; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h index 7203b80052b5f91389e1c4d3d8b85e84b46deece..db7855e69a66f65ce927b569b55ccc85e258a3c1 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -47,6 +47,15 @@ public: LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ Callable) }; + /// Used by setupJITDylib to create MachO header MaterializationUnits for + /// JITDylibs. + using MachOHeaderMUBuilder = + unique_function(MachOPlatform &MOP)>; + + /// Simple MachO header graph builder. + static inline std::unique_ptr + buildSimpleMachOHeaderMU(MachOPlatform &MOP); + /// Try to create a MachOPlatform instance, adding the ORC runtime to the /// given JITDylib. /// @@ -88,17 +97,23 @@ public: static Expected> Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, std::unique_ptr OrcRuntime, + MachOHeaderMUBuilder BuildMachOHeaderMU = buildSimpleMachOHeaderMU, std::optional RuntimeAliases = std::nullopt); /// Construct using a path to the ORC runtime. static Expected> Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, const char *OrcRuntimePath, + MachOHeaderMUBuilder BuildMachOHeaderMU = buildSimpleMachOHeaderMU, std::optional RuntimeAliases = std::nullopt); ExecutionSession &getExecutionSession() const { return ES; } ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; } + NonOwningSymbolStringPtr getMachOHeaderStartSymbol() const { + return NonOwningSymbolStringPtr(MachOHeaderStartSymbol); + } + Error setupJITDylib(JITDylib &JD) override; Error teardownJITDylib(JITDylib &JD) override; Error notifyAdding(ResourceTracker &RT, @@ -118,6 +133,9 @@ public: standardRuntimeUtilityAliases(); private: + using SymbolTableVector = SmallVector< + std::tuple>; + // Data needed for bootstrap only. struct BootstrapInfo { std::mutex Mutex; @@ -125,6 +143,7 @@ private: size_t ActiveGraphs = 0; shared::AllocActions DeferredAAs; ExecutorAddr MachOHeaderAddr; + SymbolTableVector SymTab; }; // The MachOPlatformPlugin scans/modifies LinkGraphs to support MachO @@ -239,7 +258,7 @@ private: MachOPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, std::unique_ptr OrcRuntimeGenerator, - Error &Err); + MachOHeaderMUBuilder BuildMachOHeaderMU, Error &Err); // Associate MachOPlatform JIT-side runtime support functions with handlers. Error associateRuntimeSupportFunctions(); @@ -267,6 +286,7 @@ private: ExecutionSession &ES; JITDylib &PlatformJD; ObjectLinkingLayer &ObjLinkingLayer; + MachOHeaderMUBuilder BuildMachOHeaderMU; SymbolStringPtr MachOHeaderStartSymbol = ES.intern("___dso_handle"); @@ -313,6 +333,49 @@ private: std::atomic Bootstrap; }; +// Generates a MachO header. +class SimpleMachOHeaderMU : public MaterializationUnit { +public: + SimpleMachOHeaderMU(MachOPlatform &MOP, SymbolStringPtr HeaderStartSymbol); + StringRef getName() const override { return "MachOHeaderMU"; } + void materialize(std::unique_ptr R) override; + void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override; + +protected: + virtual jitlink::Block &createHeaderBlock(JITDylib &JD, jitlink::LinkGraph &G, + jitlink::Section &HeaderSection); + +private: + struct HeaderSymbol { + const char *Name; + uint64_t Offset; + }; + + static constexpr HeaderSymbol AdditionalHeaderSymbols[] = { + {"___mh_executable_header", 0}}; + + void addMachOHeader(JITDylib &JD, jitlink::LinkGraph &G, + const SymbolStringPtr &InitializerSymbol); + static MaterializationUnit::Interface + createHeaderInterface(MachOPlatform &MOP, + const SymbolStringPtr &HeaderStartSymbol); + + MachOPlatform &MOP; +}; + +/// Simple MachO header graph builder. +inline std::unique_ptr +MachOPlatform::buildSimpleMachOHeaderMU(MachOPlatform &MOP) { + return std::make_unique(MOP, MOP.MachOHeaderStartSymbol); +} + +struct MachOHeaderInfo { + size_t PageSize = 0; + uint32_t CPUType = 0; + uint32_t CPUSubType = 0; +}; +MachOHeaderInfo getMachOHeaderInfoFromTriple(const Triple &TT); + } // end namespace orc } // end namespace llvm diff --git a/llvm/include/llvm/Frontend/HLSL/HLSLResource.h b/llvm/include/llvm/Frontend/HLSL/HLSLResource.h index ba08ee519351f739ddba3e8c11e8fc061ba7af9e..eedecaea4e58dafe3b0d0e8489900a288a61ca37 100644 --- a/llvm/include/llvm/Frontend/HLSL/HLSLResource.h +++ b/llvm/include/llvm/Frontend/HLSL/HLSLResource.h @@ -59,15 +59,16 @@ class FrontendResource { public: FrontendResource(MDNode *E) : Entry(E) { - assert(Entry->getNumOperands() == 5 && "Unexpected metadata shape"); + assert(Entry->getNumOperands() == 6 && "Unexpected metadata shape"); } FrontendResource(GlobalVariable *GV, StringRef TypeStr, ResourceKind RK, - uint32_t ResIndex, uint32_t Space); + bool IsROV, uint32_t ResIndex, uint32_t Space); GlobalVariable *getGlobalVariable(); StringRef getSourceType(); - uint32_t getResourceKind(); + ResourceKind getResourceKind(); + bool getIsROV(); uint32_t getResourceIndex(); uint32_t getSpace(); MDNode *getMetadata() { return Entry; } diff --git a/llvm/include/llvm/Frontend/Offloading/Utility.h b/llvm/include/llvm/Frontend/Offloading/Utility.h index 631d5a5a3db68e476023eb1ddfd0b617a756c268..520c192996a06645496bd55ff66d1b4d885874ea 100644 --- a/llvm/include/llvm/Frontend/Offloading/Utility.h +++ b/llvm/include/llvm/Frontend/Offloading/Utility.h @@ -6,12 +6,35 @@ // //===----------------------------------------------------------------------===// +#ifndef LLVM_FRONTEND_OFFLOADING_UTILITY_H +#define LLVM_FRONTEND_OFFLOADING_UTILITY_H + #include "llvm/IR/Module.h" #include "llvm/Object/OffloadBinary.h" namespace llvm { namespace offloading { +/// Offloading entry flags for CUDA / HIP. The first three bits indicate the +/// type of entry while the others are a bit field for additional information. +enum OffloadEntryKindFlag : uint32_t { + /// Mark the entry as a global entry. This indicates the presense of a + /// kernel if the size size field is zero and a variable otherwise. + OffloadGlobalEntry = 0x0, + /// Mark the entry as a managed global variable. + OffloadGlobalManagedEntry = 0x1, + /// Mark the entry as a surface variable. + OffloadGlobalSurfaceEntry = 0x2, + /// Mark the entry as a texture variable. + OffloadGlobalTextureEntry = 0x3, + /// Mark the entry as being extern. + OffloadGlobalExtern = 0x1 << 3, + /// Mark the entry as being constant. + OffloadGlobalConstant = 0x1 << 4, + /// Mark the entry as being a normalized surface. + OffloadGlobalNormalized = 0x1 << 5, +}; + /// Returns the type of the offloading entry we use to store kernels and /// globals that will be registered with the offloading runtime. StructType *getEntryTy(Module &M); @@ -25,7 +48,7 @@ StructType *getEntryTy(Module &M); /// char *name; // Name of the function or global. /// size_t size; // Size of the entry info (0 if it a function). /// int32_t flags; -/// int32_t reserved; +/// int32_t data; /// }; /// /// \param M The module to be used @@ -33,9 +56,11 @@ StructType *getEntryTy(Module &M); /// \param Name The symbol name associated with the global. /// \param Size The size in bytes of the global (0 for functions). /// \param Flags Flags associated with the entry. +/// \param Data Extra data storage associated with the entry. /// \param SectionName The section this entry will be placed at. void emitOffloadingEntry(Module &M, Constant *Addr, StringRef Name, - uint64_t Size, int32_t Flags, StringRef SectionName); + uint64_t Size, int32_t Flags, int32_t Data, + StringRef SectionName); /// Creates a pair of globals used to iterate the array of offloading entries by /// accessing the section variables provided by the linker. @@ -44,3 +69,5 @@ getOffloadEntryArray(Module &M, StringRef SectionName); } // namespace offloading } // namespace llvm + +#endif // LLVM_FRONTEND_OFFLOADING_UTILITY_H diff --git a/llvm/include/llvm/IR/DebugProgramInstruction.h b/llvm/include/llvm/IR/DebugProgramInstruction.h index d6b4536a2a07d5dc2dddabffaf35c6fdaf56de2c..f73a6237a47779bf91d3f64755c14164990954d2 100644 --- a/llvm/include/llvm/IR/DebugProgramInstruction.h +++ b/llvm/include/llvm/IR/DebugProgramInstruction.h @@ -113,7 +113,7 @@ public: /// Directly construct a new DPValue representing a dbg.value intrinsic /// assigning \p Location to the DV / Expr / DI variable. DPValue(Metadata *Location, DILocalVariable *DV, DIExpression *Expr, - const DILocation *DI); + const DILocation *DI, LocationType Type = LocationType::Value); /// Iterator for ValueAsMetadata that internally uses direct pointer iteration /// over either a ValueAsMetadata* or a ValueAsMetadata**, dereferencing to the diff --git a/llvm/include/llvm/IR/Dominators.h b/llvm/include/llvm/IR/Dominators.h index b6b737f34b5e33192d2e6ff2a4d776ed39fa1882..8784a425d2841a407ed9476cffbe39a4f9ae490e 100644 --- a/llvm/include/llvm/IR/Dominators.h +++ b/llvm/include/llvm/IR/Dominators.h @@ -16,7 +16,6 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/Hashing.h" @@ -24,7 +23,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Twine.h" #include "llvm/ADT/ilist_iterator.h" -#include "llvm/ADT/iterator_range.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/PassManager.h" diff --git a/llvm/include/llvm/IR/GetElementPtrTypeIterator.h b/llvm/include/llvm/IR/GetElementPtrTypeIterator.h index 8c6ede96c873bb84388dc66da2d8b5480c6ed941..f3272327c3f8b2e964d1058272cadd8b7c26eb6e 100644 --- a/llvm/include/llvm/IR/GetElementPtrTypeIterator.h +++ b/llvm/include/llvm/IR/GetElementPtrTypeIterator.h @@ -20,7 +20,6 @@ #include "llvm/IR/Operator.h" #include "llvm/IR/User.h" #include "llvm/Support/Casting.h" -#include #include #include #include diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index 06f9c0445bceac07142f2f632305512c66ca6968..bc9f99783d98f28d8d2c27b445b6d1b54a7d5b37 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -2105,7 +2105,8 @@ def int_amdgcn_wqm_vote : Intrinsic<[llvm_i1_ty], def int_amdgcn_kill : Intrinsic<[], [llvm_i1_ty], [IntrNoCallback, IntrNoFree]>; def int_amdgcn_endpgm : ClangBuiltin<"__builtin_amdgcn_endpgm">, - Intrinsic<[], [], [IntrNoReturn, IntrCold, IntrNoMem, IntrHasSideEffects, IntrNoCallback, IntrNoFree] + Intrinsic<[], [], [IntrNoReturn, IntrCold, IntrNoMem, IntrHasSideEffects, IntrConvergent, + IntrNoCallback, IntrNoFree] >; // If false, mark all active lanes as helper lanes until the end of program. diff --git a/llvm/include/llvm/IR/OperandTraits.h b/llvm/include/llvm/IR/OperandTraits.h index 979ad35019f8c79570804b67f63d5740d3fe13b4..ffece6324aab0275d15b6fc5970ece00d3fd6bfc 100644 --- a/llvm/include/llvm/IR/OperandTraits.h +++ b/llvm/include/llvm/IR/OperandTraits.h @@ -94,10 +94,10 @@ struct VariadicOperandTraits { template struct HungoffOperandTraits { static Use *op_begin(User* U) { - return U->getOperandList(); + return U->getHungOffOperands(); } static Use *op_end(User* U) { - return U->getOperandList() + U->getNumOperands(); + return U->getHungOffOperands() + U->getNumOperands(); } static unsigned operands(const User *U) { return U->getNumOperands(); diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index 07f950a9f452af458a7b66685231967f3ff91d14..096d1688af3f727c274e928f41ad3d59bc20d597 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -1270,6 +1270,14 @@ inline DisjointOr_match m_c_DisjointOr(const LHS &L, return DisjointOr_match(L, R); } +/// Match either "and" or "or disjoint". +template +inline match_combine_or, + DisjointOr_match> +m_AddLike(const LHS &L, const RHS &R) { + return m_CombineOr(m_Add(L, R), m_DisjointOr(L, R)); +} + //===----------------------------------------------------------------------===// // Class that matches a group of binary opcodes. // diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h index ebb33d0ab61f4a0f9defbf09854416818c7877b7..8931e8cab2fa187f8d970976ba817604edec12d7 100644 --- a/llvm/include/llvm/MC/MCAsmBackend.h +++ b/llvm/include/llvm/MC/MCAsmBackend.h @@ -101,7 +101,8 @@ public: /// Hook to check if a relocation is needed for some target specific reason. virtual bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo *STI) { return false; } @@ -124,7 +125,8 @@ public: virtual bool evaluateTargetFixup(const MCAssembler &Asm, const MCAsmLayout &Layout, const MCFixup &Fixup, const MCFragment *DF, - const MCValue &Target, uint64_t &Value, + const MCValue &Target, + const MCSubtargetInfo *STI, uint64_t &Value, bool &WasForced) { llvm_unreachable("Need to implement hook if target has custom fixups"); } diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h index 5e1fc738b1dab4bb962899a87ca5c103edc629f3..5ae5f6d7093858588195f2c147c53514fab31bca 100644 --- a/llvm/include/llvm/MC/MCAssembler.h +++ b/llvm/include/llvm/MC/MCAssembler.h @@ -185,7 +185,8 @@ private: /// relocation. bool evaluateFixup(const MCAsmLayout &Layout, const MCFixup &Fixup, const MCFragment *DF, MCValue &Target, - uint64_t &Value, bool &WasForced) const; + const MCSubtargetInfo *STI, uint64_t &Value, + bool &WasForced) const; /// Check whether a fixup can be satisfied, or whether it needs to be relaxed /// (increased in size, in order to hold its value correctly). @@ -221,8 +222,10 @@ private: /// finishLayout - Finalize a layout, including fragment lowering. void finishLayout(MCAsmLayout &Layout); - std::tuple - handleFixup(const MCAsmLayout &Layout, MCFragment &F, const MCFixup &Fixup); + std::tuple handleFixup(const MCAsmLayout &Layout, + MCFragment &F, + const MCFixup &Fixup, + const MCSubtargetInfo *STI); public: struct Symver { diff --git a/llvm/include/llvm/MC/MCInstrItineraries.h b/llvm/include/llvm/MC/MCInstrItineraries.h index d1c2e788ee8105c6e0cc8df61cb5084ef1d3d844..6b29686d8c4f1a118efe3f3d40c036ebc9ed8472 100644 --- a/llvm/include/llvm/MC/MCInstrItineraries.h +++ b/llvm/include/llvm/MC/MCInstrItineraries.h @@ -110,8 +110,8 @@ struct InstrItinerary { class InstrItineraryData { public: MCSchedModel SchedModel = - MCSchedModel::GetDefaultSchedModel(); ///< Basic machine properties. - const InstrStage *Stages = nullptr; ///< Array of stages selected + MCSchedModel::Default; ///< Basic machine properties. + const InstrStage *Stages = nullptr; ///< Array of stages selected const unsigned *OperandCycles = nullptr; ///< Array of operand cycles selected const unsigned *Forwardings = nullptr; ///< Array of pipeline forwarding paths const InstrItinerary *Itineraries = diff --git a/llvm/include/llvm/MC/MCSchedule.h b/llvm/include/llvm/MC/MCSchedule.h index 98ebe42cfd133b541a4a359b9f675e53db52287c..5a6471ac1c892ac7d4b579c95d618ca83205d621 100644 --- a/llvm/include/llvm/MC/MCSchedule.h +++ b/llvm/include/llvm/MC/MCSchedule.h @@ -390,7 +390,6 @@ struct MCSchedModel { unsigned WriteResourceIdx = 0); /// Returns the default initialized model. - static const MCSchedModel &GetDefaultSchedModel() { return Default; } static const MCSchedModel Default; }; diff --git a/llvm/include/llvm/MC/MCSymbolXCOFF.h b/llvm/include/llvm/MC/MCSymbolXCOFF.h index ef14b0b5c243654c296a97a5f5cabb5ec9495e8b..11c3b8831ba517642600fe20b516aade3ee531d2 100644 --- a/llvm/include/llvm/MC/MCSymbolXCOFF.h +++ b/llvm/include/llvm/MC/MCSymbolXCOFF.h @@ -17,6 +17,9 @@ namespace llvm { class MCSectionXCOFF; class MCSymbolXCOFF : public MCSymbol { + + enum XCOFFSymbolFlags : uint16_t { SF_EHInfo = 0x0001 }; + public: MCSymbolXCOFF(const StringMapEntry *Name, bool isTemporary) : MCSymbol(SymbolKindXCOFF, Name, isTemporary) {} @@ -65,6 +68,10 @@ public: return getUnqualifiedName(); } + bool isEHInfo() const { return getFlags() & SF_EHInfo; } + + void setEHInfo() const { modifyFlags(SF_EHInfo, SF_EHInfo); } + private: std::optional StorageClass; MCSectionXCOFF *RepresentedCsect = nullptr; diff --git a/llvm/include/llvm/Object/XCOFFObjectFile.h b/llvm/include/llvm/Object/XCOFFObjectFile.h index 9492284ea93d37792248dd49ade69016d56b05cd..47926ab3ae401ade4abd54bb728a999d269fa478 100644 --- a/llvm/include/llvm/Object/XCOFFObjectFile.h +++ b/llvm/include/llvm/Object/XCOFFObjectFile.h @@ -411,13 +411,13 @@ public: return Entry64->AuxType; } -private: uint8_t getSymbolAlignmentAndType() const { return GETVALUE(SymbolAlignmentAndType); } #undef GETVALUE +private: const XCOFFCsectAuxEnt32 *Entry32 = nullptr; const XCOFFCsectAuxEnt64 *Entry64 = nullptr; }; diff --git a/llvm/include/llvm/Support/AMDGPUAddrSpace.h b/llvm/include/llvm/Support/AMDGPUAddrSpace.h new file mode 100644 index 0000000000000000000000000000000000000000..72caf8d458c7a4c87ff1a150620c0aec2c6385a8 --- /dev/null +++ b/llvm/include/llvm/Support/AMDGPUAddrSpace.h @@ -0,0 +1,83 @@ +//===---------------- AMDGPUAddrSpace.h -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// AMDGPU address space definition +/// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_AMDGPUADDRSPACE_H +#define LLVM_SUPPORT_AMDGPUADDRSPACE_H + +namespace llvm { +/// OpenCL uses address spaces to differentiate between +/// various memory regions on the hardware. On the CPU +/// all of the address spaces point to the same memory, +/// however on the GPU, each address space points to +/// a separate piece of memory that is unique from other +/// memory locations. +namespace AMDGPUAS { +enum : unsigned { + // The maximum value for flat, generic, local, private, constant and region. + MAX_AMDGPU_ADDRESS = 8, + + FLAT_ADDRESS = 0, ///< Address space for flat memory. + GLOBAL_ADDRESS = 1, ///< Address space for global memory (RAT0, VTX0). + REGION_ADDRESS = 2, ///< Address space for region memory. (GDS) + + CONSTANT_ADDRESS = 4, ///< Address space for constant memory (VTX2). + LOCAL_ADDRESS = 3, ///< Address space for local memory. + PRIVATE_ADDRESS = 5, ///< Address space for private memory. + + CONSTANT_ADDRESS_32BIT = 6, ///< Address space for 32-bit constant memory. + + BUFFER_FAT_POINTER = 7, ///< Address space for 160-bit buffer fat pointers. + ///< Not used in backend. + + BUFFER_RESOURCE = 8, ///< Address space for 128-bit buffer resources. + + /// Internal address spaces. Can be freely renumbered. + STREAMOUT_REGISTER = 128, ///< Address space for GS NGG Streamout registers. + /// end Internal address spaces. + + /// Address space for direct addressable parameter memory (CONST0). + PARAM_D_ADDRESS = 6, + /// Address space for indirect addressable parameter memory (VTX1). + PARAM_I_ADDRESS = 7, + + // Do not re-order the CONSTANT_BUFFER_* enums. Several places depend on + // this order to be able to dynamically index a constant buffer, for + // example: + // + // ConstantBufferAS = CONSTANT_BUFFER_0 + CBIdx + + CONSTANT_BUFFER_0 = 8, + CONSTANT_BUFFER_1 = 9, + CONSTANT_BUFFER_2 = 10, + CONSTANT_BUFFER_3 = 11, + CONSTANT_BUFFER_4 = 12, + CONSTANT_BUFFER_5 = 13, + CONSTANT_BUFFER_6 = 14, + CONSTANT_BUFFER_7 = 15, + CONSTANT_BUFFER_8 = 16, + CONSTANT_BUFFER_9 = 17, + CONSTANT_BUFFER_10 = 18, + CONSTANT_BUFFER_11 = 19, + CONSTANT_BUFFER_12 = 20, + CONSTANT_BUFFER_13 = 21, + CONSTANT_BUFFER_14 = 22, + CONSTANT_BUFFER_15 = 23, + + // Some places use this if the address space can't be determined. + UNKNOWN_ADDRESS_SPACE = ~0u, +}; +} // end namespace AMDGPUAS +} // end namespace llvm + +#endif // LLVM_SUPPORT_AMDGPUADDRSPACE_H diff --git a/llvm/include/llvm/Support/LLVMDriver.h b/llvm/include/llvm/Support/LLVMDriver.h index 1c68f507077708fce9701ce0793e9ff739758800..0b2e265d50b423acefdf204cff9385a2a4fb460f 100644 --- a/llvm/include/llvm/Support/LLVMDriver.h +++ b/llvm/include/llvm/Support/LLVMDriver.h @@ -9,8 +9,6 @@ #ifndef LLVM_SUPPORT_LLVMDRIVER_H #define LLVM_SUPPORT_LLVMDRIVER_H -#include "llvm/ADT/SmallVector.h" - namespace llvm { struct ToolContext { diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 941c6d5f8cad8ce0d24a1972a4495cf76f6254ba..3824b1c66951e887ecdddcecd1fe0d596f9afe08 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -415,6 +415,9 @@ HANDLE_TARGET_OPCODE_MARKER(GENERIC_ATOMICRMW_OP_END, G_ATOMICRMW_UDEC_WRAP) // Generic atomic fence HANDLE_TARGET_OPCODE(G_FENCE) +/// Generic prefetch +HANDLE_TARGET_OPCODE(G_PREFETCH) + /// Generic conditional branch instruction. HANDLE_TARGET_OPCODE(G_BRCOND) @@ -454,7 +457,7 @@ HANDLE_TARGET_OPCODE(G_FCONSTANT) /// Generic va_start instruction. Stores to its one pointer operand. HANDLE_TARGET_OPCODE(G_VASTART) -/// Generic va_start instruction. Stores to its one pointer operand. +/// Generic va_arg instruction. Stores to its one pointer operand. HANDLE_TARGET_OPCODE(G_VAARG) // Generic sign extend diff --git a/llvm/include/llvm/Support/TypeSize.h b/llvm/include/llvm/Support/TypeSize.h index 4d9d4f400e9277eee64fa494384d7dad012c3a58..b00ebf9e8c454aaaf77b07763d51778f6db48241 100644 --- a/llvm/include/llvm/Support/TypeSize.h +++ b/llvm/include/llvm/Support/TypeSize.h @@ -15,7 +15,6 @@ #ifndef LLVM_SUPPORT_TYPESIZE_H #define LLVM_SUPPORT_TYPESIZE_H -#include "llvm/ADT/ArrayRef.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index 9a9c09d3c20d6124043b2fe33a34f475c60d415d..73e38b15bf6719af6f1cf2d50d925916a982a7eb 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -1209,6 +1209,15 @@ def G_FENCE : GenericInstruction { let hasSideEffects = true; } +// Generic opcode equivalent to the llvm.prefetch intrinsic. +def G_PREFETCH : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins ptype0:$address, i32imm:$rw, i32imm:$locality, i32imm:$cachetype); + let hasSideEffects = true; + let mayLoad = true; + let mayStore = true; +} + //------------------------------------------------------------------------------ // Variadic ops //------------------------------------------------------------------------------ diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h index 17cafd146b0e75dc753053d2085f2ecd311ccf50..56c32fae712cef5d651ae1d6da9806d0d262da83 100644 --- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h +++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h @@ -536,10 +536,9 @@ inline constexpr CpuInfo CpuInfos[] = { {AArch64::AEK_AES, AArch64::AEK_SHA2, AArch64::AEK_DOTPROD, AArch64::AEK_FP16, AArch64::AEK_PROFILE, AArch64::AEK_RCPC, AArch64::AEK_SSBS}))}, - {"neoverse-n2", ARMV8_5A, + {"neoverse-n2", ARMV9A, (AArch64::ExtensionBitset( - {AArch64::AEK_AES, AArch64::AEK_SHA2, AArch64::AEK_SHA3, - AArch64::AEK_SM4, AArch64::AEK_BF16, AArch64::AEK_DOTPROD, + {AArch64::AEK_BF16, AArch64::AEK_DOTPROD, AArch64::AEK_FP16, AArch64::AEK_I8MM, AArch64::AEK_MTE, AArch64::AEK_SB, AArch64::AEK_SSBS, AArch64::AEK_SVE, AArch64::AEK_SVE2, AArch64::AEK_SVE2BITPERM}))}, diff --git a/llvm/include/llvm/TargetParser/ARMTargetParser.def b/llvm/include/llvm/TargetParser/ARMTargetParser.def index 546f6bef34a41fc7a67982a0a621597a386c6d7a..c520ab898cb90a3ec821eba9c906fd82f12289aa 100644 --- a/llvm/include/llvm/TargetParser/ARMTargetParser.def +++ b/llvm/include/llvm/TargetParser/ARMTargetParser.def @@ -308,6 +308,9 @@ ARM_CPU_NAME("cortex-m55", ARMV8_1MMainline, FK_FP_ARMV8_FULLFP16_D16, false, ARM_CPU_NAME("cortex-m85", ARMV8_1MMainline, FK_FP_ARMV8_FULLFP16_D16, false, (ARM::AEK_DSP | ARM::AEK_SIMD | ARM::AEK_FP | ARM::AEK_FP16 | ARM::AEK_RAS | ARM::AEK_PACBTI)) +ARM_CPU_NAME("cortex-m52", ARMV8_1MMainline, FK_FP_ARMV8_FULLFP16_D16, false, + (ARM::AEK_DSP | ARM::AEK_SIMD | ARM::AEK_FP | ARM::AEK_FP16 | + ARM::AEK_RAS | ARM::AEK_PACBTI)) ARM_CPU_NAME("cortex-a32", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("cortex-a35", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("cortex-a53", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) @@ -337,7 +340,7 @@ ARM_CPU_NAME("cortex-x1c", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) ARM_CPU_NAME("neoverse-n1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) -ARM_CPU_NAME("neoverse-n2", ARMV8_5A, FK_CRYPTO_NEON_FP_ARMV8, false, +ARM_CPU_NAME("neoverse-n2", ARMV9A, FK_NEON_FP_ARMV8, false, (ARM::AEK_BF16 | ARM::AEK_DOTPROD | ARM::AEK_I8MM | ARM::AEK_RAS | ARM::AEK_SB)) ARM_CPU_NAME("neoverse-v1", ARMV8_4A, FK_CRYPTO_NEON_FP_ARMV8, false, diff --git a/llvm/include/llvm/TextAPI/RecordsSlice.h b/llvm/include/llvm/TextAPI/RecordsSlice.h index 779321c96efdc7b02ebfd9921763ddb6238f51e5..8d733fd797ec5160aeacd43c02f219137e3f27e0 100644 --- a/llvm/include/llvm/TextAPI/RecordsSlice.h +++ b/llvm/include/llvm/TextAPI/RecordsSlice.h @@ -128,7 +128,7 @@ public: bool hasBinaryAttrs() const { return BA.get(); } // Determine if record slice is unassigned. - bool isEmpty() const { + bool empty() const { return !hasBinaryAttrs() && Globals.empty() && Classes.empty() && Categories.empty(); } @@ -141,7 +141,7 @@ public: StringRef InstallName; StringRef UUID; StringRef Path; - FileType fileType = FileType::Invalid; + FileType File = FileType::Invalid; llvm::MachO::PackedVersion CurrentVersion; llvm::MachO::PackedVersion CompatVersion; uint8_t SwiftABI = 0; @@ -153,6 +153,10 @@ public: /// Return reference to BinaryAttrs. BinaryAttrs &getBinaryAttrs(); + /// Store any strings owned by RecordSlice into allocator and return back + /// reference to that. + StringRef copyString(StringRef String); + private: const llvm::Triple TargetTriple; // Hold tapi converted triple to avoid unecessary casts. @@ -160,7 +164,6 @@ private: /// BumpPtrAllocator to store generated/copied strings. llvm::BumpPtrAllocator StringAllocator; - StringRef copyString(StringRef String); /// Promote linkage of requested record. It is no-op if linkage type is lower /// than the current assignment. diff --git a/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h b/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h index 9df093d8d5d59777344157536c56e60dca7246f7..5ff38bdf04812b8f4e7059d0836853a200785d34 100644 --- a/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h +++ b/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h @@ -23,7 +23,6 @@ namespace llvm { class Module; -class ModuleAnaysisManager; class HipStdParAcceleratorCodeSelectionPass : public PassInfoMixin { diff --git a/llvm/include/llvm/Transforms/IPO/BlockExtractor.h b/llvm/include/llvm/Transforms/IPO/BlockExtractor.h index 6211027bd672a4c8f76978193578baf63605b899..cf6b1666b4fc6dec6fbe3ef8ffa45e585e51358e 100644 --- a/llvm/include/llvm/Transforms/IPO/BlockExtractor.h +++ b/llvm/include/llvm/Transforms/IPO/BlockExtractor.h @@ -16,7 +16,6 @@ #include -#include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" namespace llvm { diff --git a/llvm/include/llvm/Transforms/IPO/EmbedBitcodePass.h b/llvm/include/llvm/Transforms/IPO/EmbedBitcodePass.h index c35048c91aba20702453d4ebddb6514681bded3e..2bb7d5f1fcf1493a09ffee73c0ccc3e116cdcf23 100644 --- a/llvm/include/llvm/Transforms/IPO/EmbedBitcodePass.h +++ b/llvm/include/llvm/Transforms/IPO/EmbedBitcodePass.h @@ -22,7 +22,6 @@ namespace llvm { class Module; -class ModulePass; class Pass; /// Pass embeds a copy of the module optimized with the provided pass pipeline diff --git a/llvm/include/llvm/Transforms/Instrumentation.h b/llvm/include/llvm/Transforms/Instrumentation.h index 3035cdeeacf68ab840fa970c3b4a8b1958a159d7..ea97ab2562a5b0a2d0d22ba698f8946d05e9b8b7 100644 --- a/llvm/include/llvm/Transforms/Instrumentation.h +++ b/llvm/include/llvm/Transforms/Instrumentation.h @@ -49,6 +49,12 @@ GlobalVariable *createPrivateGlobalForString(Module &M, StringRef Str, // Returns nullptr on failure. Comdat *getOrCreateFunctionComdat(Function &F, Triple &T); +// Place global in a large section for x86-64 ELF binaries to mitigate +// relocation overflow pressure. This can be be used for metadata globals that +// aren't directly accessed by code, which has no performance impact. +void setGlobalVariableLargeSection(const Triple &TargetTriple, + GlobalVariable &GV); + // Insert GCOV profiling instrumentation struct GCOVOptions { static GCOVOptions getDefault(); diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h index c106e1651e8045f669a9d6d14659a308db367b31..95eb3019eab00c3a23b461d418d39c50878c26da 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h +++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h @@ -31,31 +31,46 @@ using LoadStorePair = std::pair; /// Instrumentation based profiling lowering pass. This pass lowers /// the profile instrumented code generated by FE or the IR based /// instrumentation pass. -class InstrProfiling : public PassInfoMixin { +class InstrProfilingLoweringPass + : public PassInfoMixin { + const InstrProfOptions Options = {}; + // Is this lowering for the context-sensitive instrumentation. + const bool IsCS = false; + public: - InstrProfiling() : IsCS(false) {} - InstrProfiling(const InstrProfOptions &Options, bool IsCS = false) + InstrProfilingLoweringPass() = default; + InstrProfilingLoweringPass(const InstrProfOptions &Options, bool IsCS = false) : Options(Options), IsCS(IsCS) {} PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); - bool run(Module &M, - std::function GetTLI); +}; + +class InstrProfiling final { +public: + InstrProfiling(Module &M, const InstrProfOptions &Options, + std::function GetTLI, + bool IsCS) + : M(M), Options(Options), TT(Triple(M.getTargetTriple())), IsCS(IsCS), + GetTLI(GetTLI) {} + + bool lower(); private: - InstrProfOptions Options; - Module *M; - Triple TT; + Module &M; + const InstrProfOptions Options; + const Triple TT; + // Is this lowering for the context-sensitive instrumentation. + const bool IsCS; + std::function GetTLI; struct PerFunctionProfileData { - uint32_t NumValueSites[IPVK_Last + 1]; + uint32_t NumValueSites[IPVK_Last + 1] = {}; GlobalVariable *RegionCounters = nullptr; GlobalVariable *DataVar = nullptr; GlobalVariable *RegionBitmaps = nullptr; uint32_t NumBitmapBytes = 0; - PerFunctionProfileData() { - memset(NumValueSites, 0, sizeof(uint32_t) * (IPVK_Last + 1)); - } + PerFunctionProfileData() = default; }; DenseMap ProfileDataMap; /// If runtime relocation is enabled, this maps functions to the load @@ -64,11 +79,8 @@ private: std::vector CompilerUsedVars; std::vector UsedVars; std::vector ReferencedNames; - GlobalVariable *NamesVar; - size_t NamesSize; - - // Is this lowering for the context-sensitive instrumentation. - bool IsCS; + GlobalVariable *NamesVar = nullptr; + size_t NamesSize = 0; // vector of counter load/store pairs to be register promoted. std::vector PromotionCandidates; diff --git a/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h b/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h index 293133b29cd9f9622845f5f84e427c5dc637f225..f92c6b4775a2a2f6194cd76e892bed32d8103d64 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h +++ b/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h @@ -17,9 +17,7 @@ namespace llvm { class Function; -class FunctionPass; class Module; -class ModulePass; namespace vfs { class FileSystem; diff --git a/llvm/include/llvm/Transforms/Scalar/Reassociate.h b/llvm/include/llvm/Transforms/Scalar/Reassociate.h index 28794d27325adec766ba4085cf299717299d90c1..7e47f8ae5d81e96ef217b2af1f33e6e31f68893b 100644 --- a/llvm/include/llvm/Transforms/Scalar/Reassociate.h +++ b/llvm/include/llvm/Transforms/Scalar/Reassociate.h @@ -102,7 +102,8 @@ private: void canonicalizeOperands(Instruction *I); void ReassociateExpression(BinaryOperator *I); void RewriteExprTree(BinaryOperator *I, - SmallVectorImpl &Ops); + SmallVectorImpl &Ops, + bool HasNUW); Value *OptimizeExpression(BinaryOperator *I, SmallVectorImpl &Ops); Value *OptimizeAdd(Instruction *I, diff --git a/llvm/include/llvm/Transforms/Scalar/Scalarizer.h b/llvm/include/llvm/Transforms/Scalar/Scalarizer.h index c2d9151b497107566b10d68375d090df515671c5..45e25cbf282149c65453b4d1dedb107390906b63 100644 --- a/llvm/include/llvm/Transforms/Scalar/Scalarizer.h +++ b/llvm/include/llvm/Transforms/Scalar/Scalarizer.h @@ -24,7 +24,6 @@ namespace llvm { class Function; -class FunctionPass; struct ScalarizerPassOptions { // These options correspond 1:1 to cl::opt options defined in diff --git a/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h b/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h index 417e9668527bd1d85d685f7f4684653177164339..b97ee23fc0e65d401a7155091bf17bd57aa8e45f 100644 --- a/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h +++ b/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h @@ -17,7 +17,6 @@ namespace llvm { class LPMUpdater; class Loop; -class Pass; class StringRef; class raw_ostream; diff --git a/llvm/include/llvm/Transforms/Utils/ValueMapper.h b/llvm/include/llvm/Transforms/Utils/ValueMapper.h index eedd25f898c016cca3032652f1a2ad4d45e94cff..e1f2796d97ceb18457cab193749697487f13b044 100644 --- a/llvm/include/llvm/Transforms/Utils/ValueMapper.h +++ b/llvm/include/llvm/Transforms/Utils/ValueMapper.h @@ -22,7 +22,6 @@ namespace llvm { class Constant; -class DIBuilder; class DPValue; class Function; class GlobalVariable; diff --git a/llvm/lib/Analysis/AliasSetTracker.cpp b/llvm/lib/Analysis/AliasSetTracker.cpp index 91b889116dfa2d3d5e850255ae16b0145b7ee780..debdd328ce53fdc8797b2e38737c6b632c6a9d01 100644 --- a/llvm/lib/Analysis/AliasSetTracker.cpp +++ b/llvm/lib/Analysis/AliasSetTracker.cpp @@ -348,8 +348,16 @@ AliasSet &AliasSetTracker::getAliasSetFor(const MemoryLocation &MemLoc) { // due to a quirk of alias analysis behavior. Since alias(undef, undef) // is NoAlias, mergeAliasSetsForPointer(undef, ...) will not find the // the right set for undef, even if it exists. - if (Entry.updateSizeAndAAInfo(Size, AAInfo)) + if (Entry.updateSizeAndAAInfo(Size, AAInfo)) { mergeAliasSetsForPointer(Pointer, Size, AAInfo, MustAliasAll); + + // For MustAlias sets, also update Size/AAInfo of the representative + // pointer. + AliasSet &AS = *Entry.getAliasSet(*this); + if (AS.isMustAlias()) + if (AliasSet::PointerRec *P = AS.getSomePointer()) + P->updateSizeAndAAInfo(Size, AAInfo); + } // Return the set! return *Entry.getAliasSet(*this)->getForwardedTarget(*this); } diff --git a/llvm/lib/Analysis/DomConditionCache.cpp b/llvm/lib/Analysis/DomConditionCache.cpp index 351881fe9e1f91cdc908fb15f21f437357ce6631..c7f4cab415888059a4aba272cb2cb0a4cb6e1b7c 100644 --- a/llvm/lib/Analysis/DomConditionCache.cpp +++ b/llvm/lib/Analysis/DomConditionCache.cpp @@ -36,8 +36,7 @@ static void findAffectedValues(Value *Cond, ICmpInst::Predicate Pred; Value *A; - Constant *C; - if (match(Cond, m_ICmp(Pred, m_Value(A), m_Constant(C)))) { + if (match(Cond, m_ICmp(Pred, m_Value(A), m_Constant()))) { AddAffected(A); if (ICmpInst::isEquality(Pred)) { diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp index 46629e381bc366540a11ecfaf99dcb1f9482e84d..1aa324c6b5f380c1a4be88f76e405589de85662c 100644 --- a/llvm/lib/Analysis/IVDescriptors.cpp +++ b/llvm/lib/Analysis/IVDescriptors.cpp @@ -24,8 +24,6 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/KnownBits.h" -#include - using namespace llvm; using namespace llvm::PatternMatch; diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index ee4f97f3bf5e0f4b8f1334176d9606fa50aed960..5445746ab2a1bce6472c37e4eb32c36bca753816 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -616,11 +616,11 @@ static bool isKnownNonZeroFromAssume(const Value *V, const SimplifyQuery &Q) { static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred, Value *LHS, Value *RHS, KnownBits &Known, - unsigned Depth, const SimplifyQuery &Q) { + const SimplifyQuery &Q) { if (RHS->getType()->isPointerTy()) { // Handle comparison of pointer to null explicitly, as it will not be // covered by the m_APInt() logic below. - if (match(RHS, m_Zero())) { + if (LHS == V && match(RHS, m_Zero())) { switch (Pred) { case ICmpInst::ICMP_EQ: Known.setAllZero(); @@ -720,13 +720,13 @@ void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known, BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(0)); if (Q.DT->dominates(Edge0, Q.CxtI->getParent())) computeKnownBitsFromCmp(V, Cmp->getPredicate(), Cmp->getOperand(0), - Cmp->getOperand(1), Known, Depth, Q); + Cmp->getOperand(1), Known, Q); BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(1)); if (Q.DT->dominates(Edge1, Q.CxtI->getParent())) computeKnownBitsFromCmp(V, Cmp->getInversePredicate(), Cmp->getOperand(0), Cmp->getOperand(1), Known, - Depth, Q); + Q); } if (Known.hasConflict()) @@ -794,7 +794,7 @@ void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known, continue; computeKnownBitsFromCmp(V, Cmp->getPredicate(), Cmp->getOperand(0), - Cmp->getOperand(1), Known, Depth, Q); + Cmp->getOperand(1), Known, Q); } // Conflicting assumption: Undefined behavior will occur on this execution @@ -1465,8 +1465,10 @@ static void computeKnownBitsFromOperator(const Operator *I, Q.IIQ.getMetadata(cast(I), LLVMContext::MD_range)) computeKnownBitsFromRangeMetadata(*MD, Known); if (const Value *RV = cast(I)->getReturnedArgOperand()) { - computeKnownBits(RV, Known2, Depth + 1, Q); - Known = Known.unionWith(Known2); + if (RV->getType() == I->getType()) { + computeKnownBits(RV, Known2, Depth + 1, Q); + Known = Known.unionWith(Known2); + } } if (const IntrinsicInst *II = dyn_cast(I)) { switch (II->getIntrinsicID()) { @@ -2712,7 +2714,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I, if (const auto *RP = getArgumentAliasingToReturnedPointer(Call, true)) return isKnownNonZero(RP, Depth, Q); } else if (const Value *RV = cast(I)->getReturnedArgOperand()) { - if (isKnownNonZero(RV, Depth, Q)) + if (RV->getType() == I->getType() && isKnownNonZero(RV, Depth, Q)) return true; } @@ -8024,7 +8026,7 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, for (unsigned i = 0; i != 2; ++i) { Value *L = P->getIncomingValue(i); Value *R = P->getIncomingValue(!i); - Operator *LU = dyn_cast(L); + auto *LU = dyn_cast(L); if (!LU) continue; unsigned Opcode = LU->getOpcode(); @@ -8062,7 +8064,7 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, // OR // %iv = [R, %entry], [%iv.next, %backedge] // %iv.next = binop L, %iv - BO = cast(LU); + BO = LU; Start = R; Step = L; return true; diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp index bf01b39e6f971ece5a9046ca460e64acd94bf35a..919c69fe2783e75e183ed5a5a8b6fe0fbaa7bc5a 100644 --- a/llvm/lib/AsmParser/LLLexer.cpp +++ b/llvm/lib/AsmParser/LLLexer.cpp @@ -698,6 +698,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(uinc_wrap); KEYWORD(udec_wrap); + KEYWORD(splat); KEYWORD(vscale); KEYWORD(x); KEYWORD(blockaddress); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index 5aa00225170b3a01253330b06bef1b392bd5c4c2..fb9e1ba875e1fa25470e5911370644697663b542 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -3981,6 +3981,21 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { return false; } + case lltok::kw_splat: { + Lex.Lex(); + if (parseToken(lltok::lparen, "expected '(' after vector splat")) + return true; + Constant *C; + if (parseGlobalTypeAndValue(C)) + return true; + if (parseToken(lltok::rparen, "expected ')' at end of vector splat")) + return true; + + ID.ConstantVal = C; + ID.Kind = ValID::t_ConstantSplat; + return false; + } + case lltok::kw_getelementptr: case lltok::kw_shufflevector: case lltok::kw_insertelement: @@ -5824,6 +5839,17 @@ bool LLParser::convertValIDToValue(Type *Ty, ValID &ID, Value *&V, "' but expected '" + getTypeString(Ty) + "'"); V = ID.ConstantVal; return false; + case ValID::t_ConstantSplat: + if (!Ty->isVectorTy()) + return error(ID.Loc, "vector constant must have vector type"); + if (ID.ConstantVal->getType() != Ty->getScalarType()) + return error(ID.Loc, "constant expression type mismatch: got type '" + + getTypeString(ID.ConstantVal->getType()) + + "' but expected '" + + getTypeString(Ty->getScalarType()) + "'"); + V = ConstantVector::getSplat(cast(Ty)->getElementCount(), + ID.ConstantVal); + return false; case ValID::t_ConstantStruct: case ValID::t_PackedConstantStruct: if (StructType *ST = dyn_cast(Ty)) { @@ -5861,6 +5887,7 @@ bool LLParser::parseConstantValue(Type *Ty, Constant *&C) { case ValID::t_APFloat: case ValID::t_Undef: case ValID::t_Constant: + case ValID::t_ConstantSplat: case ValID::t_ConstantStruct: case ValID::t_PackedConstantStruct: { Value *V; @@ -8622,9 +8649,9 @@ static void resolveFwdRef(ValueInfo *Fwd, ValueInfo &Resolved) { /// Stores the given Name/GUID and associated summary into the Index. /// Also updates any forward references to the associated entry ID. -void LLParser::addGlobalValueToIndex( +bool LLParser::addGlobalValueToIndex( std::string Name, GlobalValue::GUID GUID, GlobalValue::LinkageTypes Linkage, - unsigned ID, std::unique_ptr Summary) { + unsigned ID, std::unique_ptr Summary, LocTy Loc) { // First create the ValueInfo utilizing the Name or GUID. ValueInfo VI; if (GUID != 0) { @@ -8634,7 +8661,9 @@ void LLParser::addGlobalValueToIndex( assert(!Name.empty()); if (M) { auto *GV = M->getNamedValue(Name); - assert(GV); + if (!GV) + return error(Loc, "Reference to undefined global \"" + Name + "\""); + VI = Index->getOrInsertValueInfo(GV); } else { assert( @@ -8682,6 +8711,8 @@ void LLParser::addGlobalValueToIndex( NumberedValueInfos.resize(ID + 1); NumberedValueInfos[ID] = VI; } + + return false; } /// parseSummaryIndexFlags @@ -8728,6 +8759,7 @@ bool LLParser::parseGVEntry(unsigned ID) { parseToken(lltok::lparen, "expected '(' here")) return true; + LocTy Loc = Lex.getLoc(); std::string Name; GlobalValue::GUID GUID = 0; switch (Lex.getKind()) { @@ -8757,9 +8789,8 @@ bool LLParser::parseGVEntry(unsigned ID) { // an external definition. We pass ExternalLinkage since that is only // used when the GUID must be computed from Name, and in that case // the symbol must have external linkage. - addGlobalValueToIndex(Name, GUID, GlobalValue::ExternalLinkage, ID, - nullptr); - return false; + return addGlobalValueToIndex(Name, GUID, GlobalValue::ExternalLinkage, ID, + nullptr, Loc); } // Have a list of summaries @@ -8800,6 +8831,7 @@ bool LLParser::parseGVEntry(unsigned ID) { /// [',' OptionalRefs]? ')' bool LLParser::parseFunctionSummary(std::string Name, GlobalValue::GUID GUID, unsigned ID) { + LocTy Loc = Lex.getLoc(); assert(Lex.getKind() == lltok::kw_function); Lex.Lex(); @@ -8876,10 +8908,9 @@ bool LLParser::parseFunctionSummary(std::string Name, GlobalValue::GUID GUID, FS->setModulePath(ModulePath); - addGlobalValueToIndex(Name, GUID, (GlobalValue::LinkageTypes)GVFlags.Linkage, - ID, std::move(FS)); - - return false; + return addGlobalValueToIndex(Name, GUID, + (GlobalValue::LinkageTypes)GVFlags.Linkage, ID, + std::move(FS), Loc); } /// VariableSummary @@ -8887,6 +8918,7 @@ bool LLParser::parseFunctionSummary(std::string Name, GlobalValue::GUID GUID, /// [',' OptionalRefs]? ')' bool LLParser::parseVariableSummary(std::string Name, GlobalValue::GUID GUID, unsigned ID) { + LocTy Loc = Lex.getLoc(); assert(Lex.getKind() == lltok::kw_variable); Lex.Lex(); @@ -8934,10 +8966,9 @@ bool LLParser::parseVariableSummary(std::string Name, GlobalValue::GUID GUID, GS->setModulePath(ModulePath); GS->setVTableFuncs(std::move(VTableFuncs)); - addGlobalValueToIndex(Name, GUID, (GlobalValue::LinkageTypes)GVFlags.Linkage, - ID, std::move(GS)); - - return false; + return addGlobalValueToIndex(Name, GUID, + (GlobalValue::LinkageTypes)GVFlags.Linkage, ID, + std::move(GS), Loc); } /// AliasSummary @@ -8984,10 +9015,9 @@ bool LLParser::parseAliasSummary(std::string Name, GlobalValue::GUID GUID, AS->setAliasee(AliaseeVI, Summary); } - addGlobalValueToIndex(Name, GUID, (GlobalValue::LinkageTypes)GVFlags.Linkage, - ID, std::move(AS)); - - return false; + return addGlobalValueToIndex(Name, GUID, + (GlobalValue::LinkageTypes)GVFlags.Linkage, ID, + std::move(AS), Loc); } /// Flag diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 9d7e838733b5a052d637f93f9647df899bd34120..a11690a19a0e698e8a185869381e0f3aab3dcf1d 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -5248,7 +5248,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { return error( "Invalid record: operand number exceeded available operands"); - unsigned PredVal = Record[OpNum]; + CmpInst::Predicate PredVal = CmpInst::Predicate(Record[OpNum]); bool IsFP = LHS->getType()->isFPOrFPVectorTy(); FastMathFlags FMF; if (IsFP && Record.size() > OpNum+1) @@ -5257,10 +5257,15 @@ Error BitcodeReader::parseFunctionBody(Function *F) { if (OpNum+1 != Record.size()) return error("Invalid record"); - if (LHS->getType()->isFPOrFPVectorTy()) - I = new FCmpInst((FCmpInst::Predicate)PredVal, LHS, RHS); - else - I = new ICmpInst((ICmpInst::Predicate)PredVal, LHS, RHS); + if (IsFP) { + if (!CmpInst::isFPPredicate(PredVal)) + return error("Invalid fcmp predicate"); + I = new FCmpInst(PredVal, LHS, RHS); + } else { + if (!CmpInst::isIntPredicate(PredVal)) + return error("Invalid icmp predicate"); + I = new ICmpInst(PredVal, LHS, RHS); + } ResTypeID = getVirtualTypeID(I->getType()->getScalarType()); if (LHS->getType()->isVectorTy()) @@ -5363,6 +5368,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { Type *TokenTy = Type::getTokenTy(Context); Value *ParentPad = getValue(Record, Idx++, NextValueNo, TokenTy, getVirtualTypeID(TokenTy), CurBB); + if (!ParentPad) + return error("Invalid record"); unsigned NumHandlers = Record[Idx++]; @@ -5404,6 +5411,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { Type *TokenTy = Type::getTokenTy(Context); Value *ParentPad = getValue(Record, Idx++, NextValueNo, TokenTy, getVirtualTypeID(TokenTy), CurBB); + if (!ParentPad) + return error("Invald record"); unsigned NumArgOperands = Record[Idx++]; @@ -5958,6 +5967,9 @@ Error BitcodeReader::parseFunctionBody(Function *F) { if (!Align) Align = DL.getPrefTypeAlign(Ty); + if (!Size->getType()->isIntegerTy()) + return error("alloca element count must have integer type"); + AllocaInst *AI = new AllocaInst(Ty, AS, Size, *Align); AI->setUsedWithInAlloca(InAlloca); AI->setSwiftError(SwiftError); diff --git a/llvm/lib/Bitcode/Reader/ValueList.cpp b/llvm/lib/Bitcode/Reader/ValueList.cpp index b9dbf904c89e65b302e37770c62ffd358ff70199..f5568a923b11f12fa6ae8d4b7eb5cb821fff2f23 100644 --- a/llvm/lib/Bitcode/Reader/ValueList.cpp +++ b/llvm/lib/Bitcode/Reader/ValueList.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "ValueList.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/IR/Argument.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" diff --git a/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp b/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp index f00528023c91de792138367c1c41e9d5eef16fb9..ad3ad992898785f065475ba7f5ce647a372babda 100644 --- a/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp +++ b/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp @@ -2269,14 +2269,14 @@ static bool removeRedundantDbgLocsUsingBackwardScan(const BasicBlock *BB, FunctionVarLocsBuilder &FnVarLocs) { bool Changed = false; - SmallDenseMap VariableDefinedBits; + SmallDenseMap VariableDefinedBytes; // Scan over the entire block, not just over the instructions mapped by // FnVarLocs, because wedges in FnVarLocs may only be seperated by debug // instructions. for (const Instruction &I : reverse(*BB)) { if (!isa(I)) { // Sequence of consecutive defs ended. Clear map for the next one. - VariableDefinedBits.clear(); + VariableDefinedBytes.clear(); } // Get the location defs that start just before this instruction. @@ -2295,9 +2295,15 @@ removeRedundantDbgLocsUsingBackwardScan(const BasicBlock *BB, DebugAggregate Aggr = getAggregate(FnVarLocs.getVariable(RIt->VariableID)); uint64_t SizeInBits = Aggr.first->getSizeInBits().value_or(0); + uint64_t SizeInBytes = divideCeil(SizeInBits, 8); - if (SizeInBits == 0) { + // Cutoff for large variables to prevent expensive bitvector operations. + const uint64_t MaxSizeBytes = 2048; + + if (SizeInBytes == 0 || SizeInBytes > MaxSizeBytes) { // If the size is unknown (0) then keep this location def to be safe. + // Do the same for defs of large variables, which would be expensive + // to represent with a BitVector. NewDefsReversed.push_back(*RIt); continue; } @@ -2305,23 +2311,24 @@ removeRedundantDbgLocsUsingBackwardScan(const BasicBlock *BB, // Only keep this location definition if it is not fully eclipsed by // other definitions in this wedge that come after it - // Inert the bits the location definition defines. + // Inert the bytes the location definition defines. auto InsertResult = - VariableDefinedBits.try_emplace(Aggr, BitVector(SizeInBits)); + VariableDefinedBytes.try_emplace(Aggr, BitVector(SizeInBytes)); bool FirstDefinition = InsertResult.second; - BitVector &DefinedBits = InsertResult.first->second; + BitVector &DefinedBytes = InsertResult.first->second; DIExpression::FragmentInfo Fragment = RIt->Expr->getFragmentInfo().value_or( DIExpression::FragmentInfo(SizeInBits, 0)); bool InvalidFragment = Fragment.endInBits() > SizeInBits; + uint64_t StartInBytes = Fragment.startInBits() / 8; + uint64_t EndInBytes = divideCeil(Fragment.endInBits(), 8); - // If this defines any previously undefined bits, keep it. + // If this defines any previously undefined bytes, keep it. if (FirstDefinition || InvalidFragment || - DefinedBits.find_first_unset_in(Fragment.startInBits(), - Fragment.endInBits()) != -1) { + DefinedBytes.find_first_unset_in(StartInBytes, EndInBytes) != -1) { if (!InvalidFragment) - DefinedBits.set(Fragment.startInBits(), Fragment.endInBits()); + DefinedBytes.set(StartInBytes, EndInBytes); NewDefsReversed.push_back(*RIt); continue; } diff --git a/llvm/lib/CodeGen/DwarfEHPrepare.cpp b/llvm/lib/CodeGen/DwarfEHPrepare.cpp index a44aaf33c6a4997167aee2bbdaea204e1b4a6e51..e7eb34d8e6518cd0a2e225aed6ee21556523ad45 100644 --- a/llvm/lib/CodeGen/DwarfEHPrepare.cpp +++ b/llvm/lib/CodeGen/DwarfEHPrepare.cpp @@ -42,7 +42,7 @@ using namespace llvm; -#define DEBUG_TYPE "dwarfehprepare" +#define DEBUG_TYPE "dwarf-eh-prepare" STATISTIC(NumResumesLowered, "Number of resume calls lowered"); STATISTIC(NumCleanupLandingPadsUnreachable, diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 14a4e72152e7c43bba3b3162630ec5a315d2c26a..27a53e55f32fa30e192e269213e9c389f883f58c 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -2435,6 +2435,21 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, MIRBuilder.buildInstr(TargetOpcode::G_RESET_FPMODE, {}, {}); return true; } + case Intrinsic::prefetch: { + Value *Addr = CI.getOperand(0); + unsigned RW = cast(CI.getOperand(1))->getZExtValue(); + unsigned Locality = cast(CI.getOperand(2))->getZExtValue(); + unsigned CacheType = cast(CI.getOperand(3))->getZExtValue(); + + auto Flags = RW ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad; + auto &MMO = *MF->getMachineMemOperand(MachinePointerInfo(Addr), Flags, + LLT(), Align()); + + MIRBuilder.buildPrefetch(getOrCreateVReg(*Addr), RW, Locality, CacheType, + MMO); + + return true; + } #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ case Intrinsic::INTRINSIC: #include "llvm/IR/ConstrainedOps.def" diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 045fc78218daeff83554fc6c9a370b652926c0dc..37e7153be5720eb0a43ad7728e4207e4ce8161ac 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -3793,6 +3793,8 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) { return lowerTRUNC(MI); GISEL_VECREDUCE_CASES_NONSEQ return lowerVectorReduction(MI); + case G_VAARG: + return lowerVAArg(MI); } } @@ -7887,6 +7889,56 @@ LegalizerHelper::lowerVectorReduction(MachineInstr &MI) { return UnableToLegalize; } +static Type *getTypeForLLT(LLT Ty, LLVMContext &C); + +LegalizerHelper::LegalizeResult LegalizerHelper::lowerVAArg(MachineInstr &MI) { + MachineFunction &MF = *MI.getMF(); + const DataLayout &DL = MIRBuilder.getDataLayout(); + LLVMContext &Ctx = MF.getFunction().getContext(); + Register ListPtr = MI.getOperand(1).getReg(); + LLT PtrTy = MRI.getType(ListPtr); + + // LstPtr is a pointer to the head of the list. Get the address + // of the head of the list. + Align PtrAlignment = DL.getABITypeAlign(getTypeForLLT(PtrTy, Ctx)); + MachineMemOperand *PtrLoadMMO = MF.getMachineMemOperand( + MachinePointerInfo(), MachineMemOperand::MOLoad, PtrTy, PtrAlignment); + auto VAList = MIRBuilder.buildLoad(PtrTy, ListPtr, *PtrLoadMMO).getReg(0); + + const Align A(MI.getOperand(2).getImm()); + LLT PtrTyAsScalarTy = LLT::scalar(PtrTy.getSizeInBits()); + if (A > TLI.getMinStackArgumentAlignment()) { + Register AlignAmt = + MIRBuilder.buildConstant(PtrTyAsScalarTy, A.value() - 1).getReg(0); + auto AddDst = MIRBuilder.buildPtrAdd(PtrTy, VAList, AlignAmt); + auto AndDst = MIRBuilder.buildMaskLowPtrBits(PtrTy, AddDst, Log2(A)); + VAList = AndDst.getReg(0); + } + + // Increment the pointer, VAList, to the next vaarg + // The list should be bumped by the size of element in the current head of + // list. + Register Dst = MI.getOperand(0).getReg(); + LLT LLTTy = MRI.getType(Dst); + Type *Ty = getTypeForLLT(LLTTy, Ctx); + auto IncAmt = + MIRBuilder.buildConstant(PtrTyAsScalarTy, DL.getTypeAllocSize(Ty)); + auto Succ = MIRBuilder.buildPtrAdd(PtrTy, VAList, IncAmt); + + // Store the increment VAList to the legalized pointer + MachineMemOperand *StoreMMO = MF.getMachineMemOperand( + MachinePointerInfo(), MachineMemOperand::MOStore, PtrTy, PtrAlignment); + MIRBuilder.buildStore(Succ, ListPtr, *StoreMMO); + // Load the actual argument out of the pointer VAList + Align EltAlignment = DL.getABITypeAlign(Ty); + MachineMemOperand *EltLoadMMO = MF.getMachineMemOperand( + MachinePointerInfo(), MachineMemOperand::MOLoad, LLTTy, EltAlignment); + MIRBuilder.buildLoad(Dst, VAList, *EltLoadMMO); + + MI.eraseFromParent(); + return Legalized; +} + static bool shouldLowerMemFuncForSize(const MachineFunction &MF) { // On Darwin, -Os means optimize for size without hurting performance, so // only really optimize for size when -Oz (MinSize) is used. diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 80e9c08e850b6837ad9a931b5d04cb1ee3cc7127..6e1c0e907e8e4155eaa7a4356b0c0b8a5bec475b 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -1051,6 +1051,18 @@ MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) { .addImm(Scope); } +MachineInstrBuilder MachineIRBuilder::buildPrefetch(const SrcOp &Addr, + unsigned RW, + unsigned Locality, + unsigned CacheType, + MachineMemOperand &MMO) { + auto MIB = buildInstr(TargetOpcode::G_PREFETCH); + Addr.addSrcToMIB(MIB); + MIB.addImm(RW).addImm(Locality).addImm(CacheType); + MIB.addMemOperand(&MMO); + return MIB; +} + MachineInstrBuilder MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) { #ifndef NDEBUG diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp index 65a6859a006a526995afc699ee306505994bdb04..2a0daf404c97830f36a83fe6e54b30a5384ce480 100644 --- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp +++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp @@ -48,6 +48,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/InterleavedAccess.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" @@ -82,22 +83,14 @@ static cl::opt LowerInterleavedAccesses( namespace { -class InterleavedAccess : public FunctionPass { -public: - static char ID; - - InterleavedAccess() : FunctionPass(ID) { - initializeInterleavedAccessPass(*PassRegistry::getPassRegistry()); - } +class InterleavedAccessImpl { + friend class InterleavedAccess; - StringRef getPassName() const override { return "Interleaved Access Pass"; } - - bool runOnFunction(Function &F) override; - - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired(); - AU.setPreservesCFG(); - } +public: + InterleavedAccessImpl() = default; + InterleavedAccessImpl(DominatorTree *DT, const TargetLowering *TLI) + : DT(DT), TLI(TLI), MaxFactor(TLI->getMaxSupportedInterleaveFactor()) {} + bool runOnFunction(Function &F); private: DominatorTree *DT = nullptr; @@ -141,10 +134,60 @@ private: LoadInst *LI); }; +class InterleavedAccess : public FunctionPass { + InterleavedAccessImpl Impl; + +public: + static char ID; + + InterleavedAccess() : FunctionPass(ID) { + initializeInterleavedAccessPass(*PassRegistry::getPassRegistry()); + } + + StringRef getPassName() const override { return "Interleaved Access Pass"; } + + bool runOnFunction(Function &F) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.setPreservesCFG(); + } +}; + } // end anonymous namespace. +PreservedAnalyses InterleavedAccessPass::run(Function &F, + FunctionAnalysisManager &FAM) { + auto *DT = &FAM.getResult(F); + auto *TLI = TM->getSubtargetImpl(F)->getTargetLowering(); + InterleavedAccessImpl Impl(DT, TLI); + bool Changed = Impl.runOnFunction(F); + + if (!Changed) + return PreservedAnalyses::all(); + + PreservedAnalyses PA; + PA.preserveSet(); + return PA; +} + char InterleavedAccess::ID = 0; +bool InterleavedAccess::runOnFunction(Function &F) { + auto *TPC = getAnalysisIfAvailable(); + if (!TPC || !LowerInterleavedAccesses) + return false; + + LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n"); + + Impl.DT = &getAnalysis().getDomTree(); + auto &TM = TPC->getTM(); + Impl.TLI = TM.getSubtargetImpl(F)->getTargetLowering(); + Impl.MaxFactor = Impl.TLI->getMaxSupportedInterleaveFactor(); + + return Impl.runOnFunction(F); +} + INITIALIZE_PASS_BEGIN(InterleavedAccess, DEBUG_TYPE, "Lower interleaved memory accesses to target specific intrinsics", false, false) @@ -228,7 +271,7 @@ static bool isReInterleaveMask(ShuffleVectorInst *SVI, unsigned &Factor, return false; } -bool InterleavedAccess::lowerInterleavedLoad( +bool InterleavedAccessImpl::lowerInterleavedLoad( LoadInst *LI, SmallVector &DeadInsts) { if (!LI->isSimple() || isa(LI->getType())) return false; @@ -334,7 +377,7 @@ bool InterleavedAccess::lowerInterleavedLoad( return true; } -bool InterleavedAccess::replaceBinOpShuffles( +bool InterleavedAccessImpl::replaceBinOpShuffles( ArrayRef BinOpShuffles, SmallVectorImpl &Shuffles, LoadInst *LI) { for (auto *SVI : BinOpShuffles) { @@ -367,7 +410,7 @@ bool InterleavedAccess::replaceBinOpShuffles( return !BinOpShuffles.empty(); } -bool InterleavedAccess::tryReplaceExtracts( +bool InterleavedAccessImpl::tryReplaceExtracts( ArrayRef Extracts, ArrayRef Shuffles) { // If there aren't any extractelement instructions to modify, there's nothing @@ -431,7 +474,7 @@ bool InterleavedAccess::tryReplaceExtracts( return true; } -bool InterleavedAccess::lowerInterleavedStore( +bool InterleavedAccessImpl::lowerInterleavedStore( StoreInst *SI, SmallVector &DeadInsts) { if (!SI->isSimple()) return false; @@ -457,7 +500,7 @@ bool InterleavedAccess::lowerInterleavedStore( return true; } -bool InterleavedAccess::lowerDeinterleaveIntrinsic( +bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic( IntrinsicInst *DI, SmallVector &DeadInsts) { LoadInst *LI = dyn_cast(DI->getOperand(0)); @@ -476,7 +519,7 @@ bool InterleavedAccess::lowerDeinterleaveIntrinsic( return true; } -bool InterleavedAccess::lowerInterleaveIntrinsic( +bool InterleavedAccessImpl::lowerInterleaveIntrinsic( IntrinsicInst *II, SmallVector &DeadInsts) { if (!II->hasOneUse()) return false; @@ -498,18 +541,7 @@ bool InterleavedAccess::lowerInterleaveIntrinsic( return true; } -bool InterleavedAccess::runOnFunction(Function &F) { - auto *TPC = getAnalysisIfAvailable(); - if (!TPC || !LowerInterleavedAccesses) - return false; - - LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n"); - - DT = &getAnalysis().getDomTree(); - auto &TM = TPC->getTM(); - TLI = TM.getSubtargetImpl(F)->getTargetLowering(); - MaxFactor = TLI->getMaxSupportedInterleaveFactor(); - +bool InterleavedAccessImpl::runOnFunction(Function &F) { // Holds dead instructions that will be erased later. SmallVector DeadInsts; bool Changed = false; diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp index c51ef33bfe54a84ed468cd68f863d3367e68e84d..886137d86f87dea0a9d1f3d9e96dcf5e85980da5 100644 --- a/llvm/lib/CodeGen/MachineScheduler.cpp +++ b/llvm/lib/CodeGen/MachineScheduler.cpp @@ -747,9 +747,9 @@ void ScheduleDAGMI::finishBlock() { ScheduleDAGInstrs::finishBlock(); } -/// enterRegion - Called back from MachineScheduler::runOnMachineFunction after -/// crossing a scheduling boundary. [begin, end) includes all instructions in -/// the region, including the boundary itself and single-instruction regions +/// enterRegion - Called back from PostMachineScheduler::runOnMachineFunction +/// after crossing a scheduling boundary. [begin, end) includes all instructions +/// in the region, including the boundary itself and single-instruction regions /// that don't get scheduled. void ScheduleDAGMI::enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, @@ -793,9 +793,9 @@ bool ScheduleDAGMI::checkSchedLimit() { } /// Per-region scheduling driver, called back from -/// MachineScheduler::runOnMachineFunction. This is a simplified driver that -/// does not consider liveness or register pressure. It is useful for PostRA -/// scheduling and potentially other custom schedulers. +/// PostMachineScheduler::runOnMachineFunction. This is a simplified driver +/// that does not consider liveness or register pressure. It is useful for +/// PostRA scheduling and potentially other custom schedulers. void ScheduleDAGMI::schedule() { LLVM_DEBUG(dbgs() << "ScheduleDAGMI::schedule starting\n"); LLVM_DEBUG(SchedImpl->dumpPolicy()); diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index aaf9bd740d1379d0794b589d3fa8475f3653c1fa..a015d9bbd2d3f57f1e3b416caae1fc51aeb78a7f 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1812,6 +1812,29 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { } break; } + case TargetOpcode::G_PREFETCH: { + const MachineOperand &AddrOp = MI->getOperand(0); + if (!AddrOp.isReg() || !MRI->getType(AddrOp.getReg()).isPointer()) { + report("addr operand must be a pointer", &AddrOp, 0); + break; + } + const MachineOperand &RWOp = MI->getOperand(1); + if (!RWOp.isImm() || (uint64_t)RWOp.getImm() >= 2) { + report("rw operand must be an immediate 0-1", &RWOp, 1); + break; + } + const MachineOperand &LocalityOp = MI->getOperand(2); + if (!LocalityOp.isImm() || (uint64_t)LocalityOp.getImm() >= 4) { + report("locality operand must be an immediate 0-3", &LocalityOp, 2); + break; + } + const MachineOperand &CacheTypeOp = MI->getOperand(3); + if (!CacheTypeOp.isImm() || (uint64_t)CacheTypeOp.getImm() >= 2) { + report("cache type operand must be an immediate 0-1", &CacheTypeOp, 3); + break; + } + break; + } case TargetOpcode::G_ASSERT_ALIGN: { if (MI->getOperand(2).getImm() < 1) report("alignment immediate must be >= 1", MI); diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 2a3425a42607e72c1045ae7fa4b0aaf04ed3d971..f35f663d6ba1b43e899843a22fb0c316cd05dfcd 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -20940,8 +20940,8 @@ SDValue DAGCombiner::replaceStoreOfFPConstant(StoreSDNode *ST) { Ptr, ST->getMemOperand()); } - if (ST->isSimple() && - TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) { + if (ST->isSimple() && TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32) && + !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) { // Many FP stores are not made apparent until after legalize, e.g. for // argument passing. Since this is so common, custom legalize the // 64-bit integer store into two 32-bit stores. @@ -22243,6 +22243,19 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) { unsigned NumElts = VecVT.getVectorNumElements(); unsigned VecEltBitWidth = VecVT.getScalarSizeInBits(); + // See if the extracted element is constant, in which case fold it if its + // a legal fp immediate. + if (IndexC && ScalarVT.isFloatingPoint()) { + APInt EltMask = APInt::getOneBitSet(NumElts, IndexC->getZExtValue()); + KnownBits KnownElt = DAG.computeKnownBits(VecOp, EltMask); + if (KnownElt.isConstant()) { + APFloat CstFP = + APFloat(DAG.EVTToAPFloatSemantics(ScalarVT), KnownElt.getConstant()); + if (TLI.isFPImmLegal(CstFP, ScalarVT)) + return DAG.getConstantFP(CstFP, DL, ScalarVT); + } + } + // TODO: These transforms should not require the 'hasOneUse' restriction, but // there are regressions on multiple targets without it. We can end up with a // mess of scalar and vector code if we reduce only part of the DAG to scalar. diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 7a54141fa711a18499482b0b7d416e85df05c17c..5e1f9fbcdde0a64cb4fd68ae37381324e926942f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -461,7 +461,8 @@ SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { ST->getOriginalAlign(), MMOFlags, AAInfo); } - if (CFP->getValueType(0) == MVT::f64) { + if (CFP->getValueType(0) == MVT::f64 && + !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) { // If this target supports 64-bit registers, do a single 64-bit store. if (TLI.isTypeLegal(MVT::i64)) { SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index 630aa4a07d7b9464bccd0f344ba9800f1e9121f4..c4605a6b9598a9c8f539106f7e180e78b8fcf65a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -2774,6 +2774,8 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { case ISD::FPOWI: case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break; + case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break; + case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break; case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break; case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break; @@ -2902,6 +2904,24 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) { return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); } +SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) { + EVT OVT = N->getValueType(0); + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); + SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); + SDLoc dl(N); + + // Promote to the larger FP type. + Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op); + + SDValue Res = DAG.getNode(N->getOpcode(), dl, + DAG.getVTList(NVT, N->getValueType(1)), Op); + + ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); + + // Convert back to FP16 as an integer. + return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); +} + SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) { EVT RVT = N->getValueType(0); EVT SVT = N->getOperand(0).getValueType(); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index e9bd54089d0627e42736c9a173d2127c3a8150d5..9d5931b44ac67cc733459fa12216a75f0da0ace4 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -738,6 +738,7 @@ private: SDValue SoftPromoteHalfRes_FCOPYSIGN(SDNode *N); SDValue SoftPromoteHalfRes_FMAD(SDNode *N); SDValue SoftPromoteHalfRes_ExpOp(SDNode *N); + SDValue SoftPromoteHalfRes_FFREXP(SDNode *N); SDValue SoftPromoteHalfRes_FP_ROUND(SDNode *N); SDValue SoftPromoteHalfRes_LOAD(SDNode *N); SDValue SoftPromoteHalfRes_SELECT(SDNode *N); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 51ae8b703e50f2e3e94e5adf159f3ced1cbf77d9..5be1892a44f6dd0b3e41f8ea171082c06b3480a4 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -2881,6 +2881,12 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, } } + // Fallback - this is a splat if all demanded elts are the same constant. + if (computeKnownBits(V, DemandedElts, Depth).isConstant()) { + UndefElts = ~DemandedElts; + return true; + } + return false; } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index ed1c96a873748ffec2c1f0c73c674915eef7afbb..4fd76d012a167d5b5751b318ecd7408b000a9576 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -7123,7 +7123,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)), /* align */ std::nullopt, Flags); - // Chain the prefetch in parallell with any pending loads, to stay out of + // Chain the prefetch in parallel with any pending loads, to stay out of // the way of later optimizations. PendingLoads.push_back(Result); Result = getRoot(); diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index 143a4951c1361b7cb5799195bf0c56ee16e0bc22..9827bd3ff4f1ba6c1df0675593f4219d016add80 100644 --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -1048,7 +1048,7 @@ MCSection *TargetLoweringObjectFileELF::getSectionForMachineBasicBlock( } else { Name += FunctionSectionName; if (TM.getUniqueBasicBlockSectionNames()) { - if (!Name.endswith(".")) + if (!Name.ends_with(".")) Name += "."; Name += MBB.getSymbol()->getName(); } else { @@ -2310,8 +2310,10 @@ bool TargetLoweringObjectFileXCOFF::ShouldSetSSPCanaryBitInTB( MCSymbol * TargetLoweringObjectFileXCOFF::getEHInfoTableSymbol(const MachineFunction *MF) { - return MF->getMMI().getContext().getOrCreateSymbol( + MCSymbol *EHInfoSym = MF->getMMI().getContext().getOrCreateSymbol( "__ehinfo." + Twine(MF->getFunctionNumber())); + cast(EHInfoSym)->setEHInfo(); + return EHInfoSym; } MCSymbol * @@ -2644,12 +2646,16 @@ MCSection *TargetLoweringObjectFileXCOFF::getSectionForFunctionDescriptor( MCSection *TargetLoweringObjectFileXCOFF::getSectionForTOCEntry( const MCSymbol *Sym, const TargetMachine &TM) const { // Use TE storage-mapping class when large code model is enabled so that - // the chance of needing -bbigtoc is decreased. + // the chance of needing -bbigtoc is decreased. Also, the toc-entry for + // EH info is never referenced directly using instructions so it can be + // allocated with TE storage-mapping class. return getContext().getXCOFFSection( cast(Sym)->getSymbolTableName(), SectionKind::getData(), - XCOFF::CsectProperties( - TM.getCodeModel() == CodeModel::Large ? XCOFF::XMC_TE : XCOFF::XMC_TC, - XCOFF::XTY_SD)); + XCOFF::CsectProperties((TM.getCodeModel() == CodeModel::Large || + cast(Sym)->isEHInfo()) + ? XCOFF::XMC_TE + : XCOFF::XMC_TC, + XCOFF::XTY_SD)); } MCSection *TargetLoweringObjectFileXCOFF::getSectionForLSDA( diff --git a/llvm/lib/CodeGen/WinEHPrepare.cpp b/llvm/lib/CodeGen/WinEHPrepare.cpp index 347bada6f69d51985f9a7aab6264e8f41162fdc1..bd14e8104a6ade315c9a46feda933fa39ab13340 100644 --- a/llvm/lib/CodeGen/WinEHPrepare.cpp +++ b/llvm/lib/CodeGen/WinEHPrepare.cpp @@ -120,8 +120,8 @@ PreservedAnalyses WinEHPreparePass::run(Function &F, } char WinEHPrepare::ID = 0; -INITIALIZE_PASS(WinEHPrepare, "winehprepare", "Prepare Windows exceptions", - false, false) +INITIALIZE_PASS(WinEHPrepare, DEBUG_TYPE, "Prepare Windows exceptions", false, + false) FunctionPass *llvm::createWinEHPass(bool DemoteCatchSwitchPHIOnly) { return new WinEHPrepare(DemoteCatchSwitchPHIOnly); diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp index 0f01933002c06db3af33f431f9b073346f7f719d..66492f7bf8043330458b9ea9453f7941d2d6a379 100644 --- a/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp +++ b/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/DebugInfo/DWARF/DWARFDie.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/Dwarf.h" @@ -489,18 +490,23 @@ void DWARFDie::getCallerFrame(uint32_t &CallFile, uint32_t &CallLine, CallDiscriminator = toUnsigned(find(DW_AT_GNU_discriminator), 0); } -std::optional DWARFDie::getTypeSize(uint64_t PointerSize) { - if (auto SizeAttr = find(DW_AT_byte_size)) +static std::optional +getTypeSizeImpl(DWARFDie Die, uint64_t PointerSize, + SmallPtrSetImpl &Visited) { + // Cycle detected? + if (!Visited.insert(Die.getDebugInfoEntry()).second) + return {}; + if (auto SizeAttr = Die.find(DW_AT_byte_size)) if (std::optional Size = SizeAttr->getAsUnsignedConstant()) return Size; - switch (getTag()) { + switch (Die.getTag()) { case DW_TAG_pointer_type: case DW_TAG_reference_type: case DW_TAG_rvalue_reference_type: return PointerSize; case DW_TAG_ptr_to_member_type: { - if (DWARFDie BaseType = getAttributeValueAsReferencedDie(DW_AT_type)) + if (DWARFDie BaseType = Die.getAttributeValueAsReferencedDie(DW_AT_type)) if (BaseType.getTag() == DW_TAG_subroutine_type) return 2 * PointerSize; return PointerSize; @@ -510,19 +516,20 @@ std::optional DWARFDie::getTypeSize(uint64_t PointerSize) { case DW_TAG_volatile_type: case DW_TAG_restrict_type: case DW_TAG_typedef: { - if (DWARFDie BaseType = getAttributeValueAsReferencedDie(DW_AT_type)) - return BaseType.getTypeSize(PointerSize); + if (DWARFDie BaseType = Die.getAttributeValueAsReferencedDie(DW_AT_type)) + return getTypeSizeImpl(BaseType, PointerSize, Visited); break; } case DW_TAG_array_type: { - DWARFDie BaseType = getAttributeValueAsReferencedDie(DW_AT_type); + DWARFDie BaseType = Die.getAttributeValueAsReferencedDie(DW_AT_type); if (!BaseType) return std::nullopt; - std::optional BaseSize = BaseType.getTypeSize(PointerSize); + std::optional BaseSize = + getTypeSizeImpl(BaseType, PointerSize, Visited); if (!BaseSize) return std::nullopt; uint64_t Size = *BaseSize; - for (DWARFDie Child : *this) { + for (DWARFDie Child : Die) { if (Child.getTag() != DW_TAG_subrange_type) continue; @@ -542,13 +549,18 @@ std::optional DWARFDie::getTypeSize(uint64_t PointerSize) { return Size; } default: - if (DWARFDie BaseType = getAttributeValueAsReferencedDie(DW_AT_type)) - return BaseType.getTypeSize(PointerSize); + if (DWARFDie BaseType = Die.getAttributeValueAsReferencedDie(DW_AT_type)) + return getTypeSizeImpl(BaseType, PointerSize, Visited); break; } return std::nullopt; } +std::optional DWARFDie::getTypeSize(uint64_t PointerSize) { + SmallPtrSet Visited; + return getTypeSizeImpl(*this, PointerSize, Visited); +} + /// Helper to dump a DIE with all of its parents, but no siblings. static unsigned dumpParentChain(DWARFDie Die, raw_ostream &OS, unsigned Indent, DIDumpOptions DumpOpts, unsigned Depth = 0) { diff --git a/llvm/lib/DebugInfo/PDB/Native/FormatUtil.cpp b/llvm/lib/DebugInfo/PDB/Native/FormatUtil.cpp index 9c05d585831a920065fc3d03c249f95fb80b5672..c5999bffc021da9ca064ea6340c3a848ad42b0f8 100644 --- a/llvm/lib/DebugInfo/PDB/Native/FormatUtil.cpp +++ b/llvm/lib/DebugInfo/PDB/Native/FormatUtil.cpp @@ -9,6 +9,7 @@ #include "llvm/DebugInfo/PDB/Native/FormatUtil.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/ADT/StringExtras.h" #include "llvm/BinaryFormat/COFF.h" #include "llvm/DebugInfo/CodeView/CodeView.h" @@ -119,9 +120,7 @@ std::string llvm::pdb::formatTypeLeafKind(TypeLeafKind K) { return #EnumName; #include "llvm/DebugInfo/CodeView/CodeViewTypes.def" default: - return formatv("UNKNOWN RECORD ({0:X})", - static_cast>(K)) - .str(); + return formatv("UNKNOWN RECORD ({0:X})", llvm::to_underlying(K)).str(); } } diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt index 23b4714816185c2946651f927c85139a813df258..5bf23a7ec0bc89206b6ccc642af3c833361b366e 100644 --- a/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt @@ -6,6 +6,7 @@ add_llvm_component_library(LLVMOrcDebugging DebugInfoSupport.cpp DebuggerSupport.cpp DebuggerSupportPlugin.cpp + LLJITUtilsCBindings.cpp PerfSupportPlugin.cpp ADDITIONAL_HEADER_DIRS diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupport.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupport.cpp index 9ba6dd90f50de51a897d723226fb8506229e6a1a..1668473c0eb47498e08e867ed670cc74c69d4ce4 100644 --- a/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupport.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupport.cpp @@ -39,7 +39,7 @@ Error enableDebuggerSupport(LLJIT &J) { if (!Registrar) return Registrar.takeError(); ObjLinkingLayer->addPlugin(std::make_unique( - ES, std::move(*Registrar), true, true)); + ES, std::move(*Registrar), false, true)); return Error::success(); } case Triple::MachO: { diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/LLJITUtilsCBindings.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/LLJITUtilsCBindings.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2df5aef733fb32c43152b51745a009c177758fa5 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/LLJITUtilsCBindings.cpp @@ -0,0 +1,22 @@ +//===--------- LLJITUtilsCBindings.cpp - Advanced LLJIT features ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm-c/LLJIT.h" +#include "llvm-c/LLJITUtils.h" + +#include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" + +using namespace llvm; +using namespace llvm::orc; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJIT, LLVMOrcLLJITRef) + +LLVMErrorRef LLVMOrcLLJITEnableDebugSupport(LLVMOrcLLJITRef J) { + return wrap(llvm::orc::enableDebuggerSupport(*unwrap(J))); +} diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index a0bd9b6266ff143c5436ec3e3dec7722bb10df8c..9057300bf04352a9ab44e3092384571a9a7c99fe 100644 --- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -15,6 +15,7 @@ #include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" +#include "llvm/ExecutionEngine/Orc/MachOBuilder.h" #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/Debug.h" @@ -89,6 +90,11 @@ public: namespace { +using SPSRegisterSymbolsArgs = + SPSArgList>>; + std::unique_ptr createPlatformGraph(MachOPlatform &MOP, std::string Name) { unsigned PointerSize; @@ -110,122 +116,32 @@ std::unique_ptr createPlatformGraph(MachOPlatform &MOP, jitlink::getGenericEdgeKindName); } -// Generates a MachO header. -class MachOHeaderMaterializationUnit : public MaterializationUnit { -public: - MachOHeaderMaterializationUnit(MachOPlatform &MOP, - const SymbolStringPtr &HeaderStartSymbol) - : MaterializationUnit(createHeaderInterface(MOP, HeaderStartSymbol)), - MOP(MOP) {} - - StringRef getName() const override { return "MachOHeaderMU"; } - - void materialize(std::unique_ptr R) override { - auto G = createPlatformGraph(MOP, ""); - addMachOHeader(*G, MOP, R->getInitializerSymbol()); - MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); - } - - void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} - - static void addMachOHeader(jitlink::LinkGraph &G, MachOPlatform &MOP, - const SymbolStringPtr &InitializerSymbol) { - auto &HeaderSection = G.createSection("__header", MemProt::Read); - auto &HeaderBlock = createHeaderBlock(G, HeaderSection); - - // Init symbol is header-start symbol. - G.addDefinedSymbol(HeaderBlock, 0, *InitializerSymbol, - HeaderBlock.getSize(), jitlink::Linkage::Strong, - jitlink::Scope::Default, false, true); - for (auto &HS : AdditionalHeaderSymbols) - G.addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, HeaderBlock.getSize(), - jitlink::Linkage::Strong, jitlink::Scope::Default, - false, true); - } - -private: - struct HeaderSymbol { - const char *Name; - uint64_t Offset; - }; - - static constexpr HeaderSymbol AdditionalHeaderSymbols[] = { - {"___mh_executable_header", 0}}; - - static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G, - jitlink::Section &HeaderSection) { - MachO::mach_header_64 Hdr; - Hdr.magic = MachO::MH_MAGIC_64; - switch (G.getTargetTriple().getArch()) { - case Triple::aarch64: - Hdr.cputype = MachO::CPU_TYPE_ARM64; - Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; - break; - case Triple::x86_64: - Hdr.cputype = MachO::CPU_TYPE_X86_64; - Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; - break; - default: - llvm_unreachable("Unrecognized architecture"); - } - Hdr.filetype = MachO::MH_DYLIB; // Custom file type? - Hdr.ncmds = 0; - Hdr.sizeofcmds = 0; - Hdr.flags = 0; - Hdr.reserved = 0; - - if (G.getEndianness() != llvm::endianness::native) - MachO::swapStruct(Hdr); - - auto HeaderContent = G.allocateContent( - ArrayRef(reinterpret_cast(&Hdr), sizeof(Hdr))); - - return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8, - 0); - } - - static MaterializationUnit::Interface - createHeaderInterface(MachOPlatform &MOP, - const SymbolStringPtr &HeaderStartSymbol) { - SymbolFlagsMap HeaderSymbolFlags; - - HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported; - for (auto &HS : AdditionalHeaderSymbols) - HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] = - JITSymbolFlags::Exported; - - return MaterializationUnit::Interface(std::move(HeaderSymbolFlags), - HeaderStartSymbol); - } - - MachOPlatform &MOP; -}; - -constexpr MachOHeaderMaterializationUnit::HeaderSymbol - MachOHeaderMaterializationUnit::AdditionalHeaderSymbols[]; - // Creates a Bootstrap-Complete LinkGraph to run deferred actions. class MachOPlatformCompleteBootstrapMaterializationUnit : public MaterializationUnit { public: + using SymbolTableVector = + SmallVector>; + MachOPlatformCompleteBootstrapMaterializationUnit( MachOPlatform &MOP, StringRef PlatformJDName, - SymbolStringPtr CompleteBootstrapSymbol, shared::AllocActions DeferredAAs, + SymbolStringPtr CompleteBootstrapSymbol, SymbolTableVector SymTab, + shared::AllocActions DeferredAAs, ExecutorAddr MachOHeaderAddr, ExecutorAddr PlatformBootstrap, ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib, ExecutorAddr DeregisterJITDylib, ExecutorAddr RegisterObjectSymbolTable, - ExecutorAddr DeregisterObjectSymbolTable, ExecutorAddr MachOHeaderAddr) + ExecutorAddr DeregisterObjectSymbolTable) : MaterializationUnit( {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}), MOP(MOP), PlatformJDName(PlatformJDName), CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)), - DeferredAAs(std::move(DeferredAAs)), - PlatformBootstrap(PlatformBootstrap), + SymTab(std::move(SymTab)), DeferredAAs(std::move(DeferredAAs)), + MachOHeaderAddr(MachOHeaderAddr), PlatformBootstrap(PlatformBootstrap), PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib), DeregisterJITDylib(DeregisterJITDylib), RegisterObjectSymbolTable(RegisterObjectSymbolTable), - DeregisterObjectSymbolTable(DeregisterObjectSymbolTable), - MachOHeaderAddr(MachOHeaderAddr) {} + DeregisterObjectSymbolTable(DeregisterObjectSymbolTable) {} StringRef getName() const override { return "MachOPlatformCompleteBootstrap"; @@ -242,7 +158,7 @@ public: Linkage::Strong, Scope::Hidden, false, true); // Reserve space for the stolen actions, plus two extras. - G->allocActions().reserve(DeferredAAs.size() + 2); + G->allocActions().reserve(DeferredAAs.size() + 3); // 1. Bootstrap the platform support code. G->allocActions().push_back( @@ -258,7 +174,14 @@ public: cantFail(WrapperFunctionCall::Create>( DeregisterJITDylib, MachOHeaderAddr))}); - // 3. Add the deferred actions to the graph. + // 3. Register deferred symbols. + G->allocActions().push_back( + {cantFail(WrapperFunctionCall::Create( + RegisterObjectSymbolTable, MachOHeaderAddr, SymTab)), + cantFail(WrapperFunctionCall::Create( + DeregisterObjectSymbolTable, MachOHeaderAddr, SymTab))}); + + // 4. Add the deferred actions to the graph. std::move(DeferredAAs.begin(), DeferredAAs.end(), std::back_inserter(G->allocActions())); @@ -271,14 +194,15 @@ private: MachOPlatform &MOP; StringRef PlatformJDName; SymbolStringPtr CompleteBootstrapSymbol; + SymbolTableVector SymTab; shared::AllocActions DeferredAAs; + ExecutorAddr MachOHeaderAddr; ExecutorAddr PlatformBootstrap; ExecutorAddr PlatformShutdown; ExecutorAddr RegisterJITDylib; ExecutorAddr DeregisterJITDylib; ExecutorAddr RegisterObjectSymbolTable; ExecutorAddr DeregisterObjectSymbolTable; - ExecutorAddr MachOHeaderAddr; }; static StringRef ObjCRuntimeObjectSectionsData[] = { @@ -335,6 +259,7 @@ Expected> MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, std::unique_ptr OrcRuntime, + MachOHeaderMUBuilder BuildMachOHeaderMU, std::optional RuntimeAliases) { // If the target is not supported then bail out immediately. @@ -365,8 +290,9 @@ MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, // Create the instance. Error Err = Error::success(); - auto P = std::unique_ptr(new MachOPlatform( - ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err)); + auto P = std::unique_ptr( + new MachOPlatform(ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), + std::move(BuildMachOHeaderMU), Err)); if (Err) return std::move(Err); return std::move(P); @@ -375,6 +301,7 @@ MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, Expected> MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, const char *OrcRuntimePath, + MachOHeaderMUBuilder BuildMachOHeaderMU, std::optional RuntimeAliases) { // Create a generator for the ORC runtime archive. @@ -385,12 +312,11 @@ MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, return Create(ES, ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator), - std::move(RuntimeAliases)); + std::move(BuildMachOHeaderMU), std::move(RuntimeAliases)); } Error MachOPlatform::setupJITDylib(JITDylib &JD) { - if (auto Err = JD.define(std::make_unique( - *this, MachOHeaderStartSymbol))) + if (auto Err = JD.define(BuildMachOHeaderMU(*this))) return Err; return ES.lookup({&JD}, MachOHeaderStartSymbol).takeError(); @@ -505,8 +431,10 @@ MachOPlatform::flagsForSymbol(jitlink::Symbol &Sym) { MachOPlatform::MachOPlatform( ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, - std::unique_ptr OrcRuntimeGenerator, Error &Err) - : ES(ES), PlatformJD(PlatformJD), ObjLinkingLayer(ObjLinkingLayer) { + std::unique_ptr OrcRuntimeGenerator, + MachOHeaderMUBuilder BuildMachOHeaderMU, Error &Err) + : ES(ES), PlatformJD(PlatformJD), ObjLinkingLayer(ObjLinkingLayer), + BuildMachOHeaderMU(std::move(BuildMachOHeaderMU)) { ErrorAsOutParameter _(&Err); ObjLinkingLayer.addPlugin(std::make_unique(*this)); PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); @@ -569,8 +497,7 @@ MachOPlatform::MachOPlatform( // the support methods callable. The bootstrap is now complete. // Step (1) Add header materialization unit and request. - if ((Err = PlatformJD.define(std::make_unique( - *this, MachOHeaderStartSymbol)))) + if ((Err = PlatformJD.define(this->BuildMachOHeaderMU(*this)))) return; if ((Err = ES.lookup(&PlatformJD, MachOHeaderStartSymbol).takeError())) return; @@ -601,10 +528,11 @@ MachOPlatform::MachOPlatform( if ((Err = PlatformJD.define( std::make_unique( *this, PlatformJD.getName(), BootstrapCompleteSymbol, - std::move(BI.DeferredAAs), PlatformBootstrap.Addr, + std::move(BI.SymTab), std::move(BI.DeferredAAs), + BI.MachOHeaderAddr, PlatformBootstrap.Addr, PlatformShutdown.Addr, RegisterJITDylib.Addr, DeregisterJITDylib.Addr, RegisterObjectSymbolTable.Addr, - DeregisterObjectSymbolTable.Addr, BI.MachOHeaderAddr)))) + DeregisterObjectSymbolTable.Addr)))) return; if ((Err = ES.lookup(makeJITDylibSearchOrder( &PlatformJD, JITDylibLookupFlags::MatchAllSymbols), @@ -1714,16 +1642,17 @@ Error MachOPlatform::MachOPlatformPlugin::addSymbolTableRegistration( HeaderAddr = I->second; } - SmallVector> - SymTab; + SymbolTableVector LocalSymTab; + auto &SymTab = LLVM_LIKELY(!InBootstrapPhase) ? LocalSymTab + : MP.Bootstrap.load()->SymTab; for (auto &[OriginalSymbol, NameSym] : JITSymTabInfo) SymTab.push_back({NameSym->getAddress(), OriginalSymbol->getAddress(), flagsForSymbol(*OriginalSymbol)}); - using SPSRegisterSymbolsArgs = - SPSArgList>>; + // Bail out if we're in the bootstrap phase -- registration of thees symbols + // will be attached to the bootstrap graph. + if (LLVM_UNLIKELY(InBootstrapPhase)) + return Error::success(); shared::AllocActions &allocActions = LLVM_LIKELY(!InBootstrapPhase) ? G.allocActions() @@ -1736,5 +1665,97 @@ Error MachOPlatform::MachOPlatformPlugin::addSymbolTableRegistration( return Error::success(); } + +template +jitlink::Block &createTrivialHeaderBlock(MachOPlatform &MOP, + jitlink::LinkGraph &G, + jitlink::Section &HeaderSection) { + auto HdrInfo = + getMachOHeaderInfoFromTriple(MOP.getExecutionSession().getTargetTriple()); + MachOBuilder B(HdrInfo.PageSize); + + B.Header.filetype = MachO::MH_DYLIB; + B.Header.cputype = HdrInfo.CPUType; + B.Header.cpusubtype = HdrInfo.CPUSubType; + + auto HeaderContent = G.allocateBuffer(B.layout()); + B.write(HeaderContent); + + return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8, + 0); +} + +SimpleMachOHeaderMU::SimpleMachOHeaderMU(MachOPlatform &MOP, + SymbolStringPtr HeaderStartSymbol) + : MaterializationUnit( + createHeaderInterface(MOP, std::move(HeaderStartSymbol))), + MOP(MOP) {} + +void SimpleMachOHeaderMU::materialize( + std::unique_ptr R) { + auto G = createPlatformGraph(MOP, ""); + addMachOHeader(R->getTargetJITDylib(), *G, R->getInitializerSymbol()); + MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); +} + +void SimpleMachOHeaderMU::discard(const JITDylib &JD, + const SymbolStringPtr &Sym) {} + +void SimpleMachOHeaderMU::addMachOHeader( + JITDylib &JD, jitlink::LinkGraph &G, + const SymbolStringPtr &InitializerSymbol) { + auto &HeaderSection = G.createSection("__header", MemProt::Read); + auto &HeaderBlock = createHeaderBlock(JD, G, HeaderSection); + + // Init symbol is header-start symbol. + G.addDefinedSymbol(HeaderBlock, 0, *InitializerSymbol, HeaderBlock.getSize(), + jitlink::Linkage::Strong, jitlink::Scope::Default, false, + true); + for (auto &HS : AdditionalHeaderSymbols) + G.addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, HeaderBlock.getSize(), + jitlink::Linkage::Strong, jitlink::Scope::Default, false, + true); +} + +jitlink::Block & +SimpleMachOHeaderMU::createHeaderBlock(JITDylib &JD, jitlink::LinkGraph &G, + jitlink::Section &HeaderSection) { + switch (MOP.getExecutionSession().getTargetTriple().getArch()) { + case Triple::aarch64: + case Triple::x86_64: + return createTrivialHeaderBlock(MOP, G, HeaderSection); + default: + llvm_unreachable("Unsupported architecture"); + } +} + +MaterializationUnit::Interface SimpleMachOHeaderMU::createHeaderInterface( + MachOPlatform &MOP, const SymbolStringPtr &HeaderStartSymbol) { + SymbolFlagsMap HeaderSymbolFlags; + + HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported; + for (auto &HS : AdditionalHeaderSymbols) + HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] = + JITSymbolFlags::Exported; + + return MaterializationUnit::Interface(std::move(HeaderSymbolFlags), + HeaderStartSymbol); +} + +MachOHeaderInfo getMachOHeaderInfoFromTriple(const Triple &TT) { + switch (TT.getArch()) { + case Triple::aarch64: + return {/* PageSize = */ 16 * 1024, + /* CPUType = */ MachO::CPU_TYPE_ARM64, + /* CPUSubType = */ MachO::CPU_SUBTYPE_ARM64_ALL}; + case Triple::x86_64: + return {/* PageSize = */ 4 * 1024, + /* CPUType = */ MachO::CPU_TYPE_X86_64, + /* CPUSubType = */ MachO::CPU_SUBTYPE_X86_64_ALL}; + default: + llvm_unreachable("Unrecognized architecture"); + } +} + } // End namespace orc. } // End namespace llvm. diff --git a/llvm/lib/Frontend/HLSL/HLSLResource.cpp b/llvm/lib/Frontend/HLSL/HLSLResource.cpp index 59f730d8a495dc545267d9aa68321f10438607ea..709fe3212623ef95503040d6a08cfe24db924e53 100644 --- a/llvm/lib/Frontend/HLSL/HLSLResource.cpp +++ b/llvm/lib/Frontend/HLSL/HLSLResource.cpp @@ -27,30 +27,37 @@ StringRef FrontendResource::getSourceType() { return cast(Entry->getOperand(1))->getString(); } -uint32_t FrontendResource::FrontendResource::getResourceKind() { +ResourceKind FrontendResource::getResourceKind() { + return static_cast( + cast( + cast(Entry->getOperand(2))->getValue()) + ->getLimitedValue()); +} +bool FrontendResource::getIsROV() { return cast( - cast(Entry->getOperand(2))->getValue()) + cast(Entry->getOperand(3))->getValue()) ->getLimitedValue(); } uint32_t FrontendResource::getResourceIndex() { return cast( - cast(Entry->getOperand(3))->getValue()) + cast(Entry->getOperand(4))->getValue()) ->getLimitedValue(); } uint32_t FrontendResource::getSpace() { return cast( - cast(Entry->getOperand(4))->getValue()) + cast(Entry->getOperand(5))->getValue()) ->getLimitedValue(); } FrontendResource::FrontendResource(GlobalVariable *GV, StringRef TypeStr, - ResourceKind RK, uint32_t ResIndex, - uint32_t Space) { + ResourceKind RK, bool IsROV, + uint32_t ResIndex, uint32_t Space) { auto &Ctx = GV->getContext(); IRBuilder<> B(Ctx); Entry = MDNode::get( Ctx, {ValueAsMetadata::get(GV), MDString::get(Ctx, TypeStr), ConstantAsMetadata::get(B.getInt32(static_cast(RK))), + ConstantAsMetadata::get(B.getInt1(IsROV)), ConstantAsMetadata::get(B.getInt32(ResIndex)), ConstantAsMetadata::get(B.getInt32(Space))}); } diff --git a/llvm/lib/Frontend/Offloading/Utility.cpp b/llvm/lib/Frontend/Offloading/Utility.cpp index 1c08f02c17f5fac48f2197b77292b99e6c14b719..25f609517ebeb7143b6ab7c58b2b61b0c57dfcd8 100644 --- a/llvm/lib/Frontend/Offloading/Utility.cpp +++ b/llvm/lib/Frontend/Offloading/Utility.cpp @@ -29,7 +29,7 @@ StructType *offloading::getEntryTy(Module &M) { // TODO: Rework this interface to be more generic. void offloading::emitOffloadingEntry(Module &M, Constant *Addr, StringRef Name, - uint64_t Size, int32_t Flags, + uint64_t Size, int32_t Flags, int32_t Data, StringRef SectionName) { llvm::Triple Triple(M.getTargetTriple()); @@ -51,7 +51,7 @@ void offloading::emitOffloadingEntry(Module &M, Constant *Addr, StringRef Name, ConstantExpr::getPointerBitCastOrAddrSpaceCast(Str, Int8PtrTy), ConstantInt::get(SizeTy, Size), ConstantInt::get(Int32Ty, Flags), - ConstantInt::get(Int32Ty, 0), + ConstantInt::get(Int32Ty, Data), }; Constant *EntryInitializer = ConstantStruct::get(getEntryTy(M), EntryData); diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index 4c1def507df46cef0255bcfacc257a529f583fa7..be9a44885f668eb06a0eb45698d33db0b8f3da34 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -6393,7 +6393,7 @@ void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr, StringRef Name) { if (!Config.isGPU()) { llvm::offloading::emitOffloadingEntry( - M, ID, Name.empty() ? Addr->getName() : Name, Size, Flags, + M, ID, Name.empty() ? Addr->getName() : Name, Size, Flags, /*Data=*/0, "omp_offloading_entries"); return; } diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index bff64e3a15a24710d5d88e8f885b7c5184dd7906..95cdec722062e382eca5fe7957216f035b11ca27 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -3520,15 +3520,15 @@ static void printMetadataIdentifier(StringRef Name, if (Name.empty()) { Out << " "; } else { - if (isalpha(static_cast(Name[0])) || Name[0] == '-' || - Name[0] == '$' || Name[0] == '.' || Name[0] == '_') - Out << Name[0]; + unsigned char FirstC = static_cast(Name[0]); + if (isalpha(FirstC) || FirstC == '-' || FirstC == '$' || FirstC == '.' || + FirstC == '_') + Out << FirstC; else - Out << '\\' << hexdigit(Name[0] >> 4) << hexdigit(Name[0] & 0x0F); + Out << '\\' << hexdigit(FirstC >> 4) << hexdigit(FirstC & 0x0F); for (unsigned i = 1, e = Name.size(); i != e; ++i) { unsigned char C = Name[i]; - if (isalnum(static_cast(C)) || C == '-' || C == '$' || - C == '.' || C == '_') + if (isalnum(C) || C == '-' || C == '$' || C == '.' || C == '_') Out << C; else Out << '\\' << hexdigit(C >> 4) << hexdigit(C & 0x0F); diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index 67ee7b7d97e9aa934b66d010350bafb886ed7e1b..645691f4417945e0f9ae05f9dcc6fb32bd74c161 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -1293,7 +1293,8 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { } auto *ST = dyn_cast(F->getReturnType()); - if (ST && (!ST->isLiteral() || ST->isPacked())) { + if (ST && (!ST->isLiteral() || ST->isPacked()) && + F->getIntrinsicID() != Intrinsic::not_intrinsic) { // Replace return type with literal non-packed struct. Only do this for // intrinsics declared to return a struct, not for intrinsics with // overloaded return type, in which case the exact struct type will be diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp index 7832028bf367119f86515531ceb3ad77d0c42cca..96629de8a7534ec59505f768422a3eadcb89d336 100644 --- a/llvm/lib/IR/Core.cpp +++ b/llvm/lib/IR/Core.cpp @@ -43,6 +43,8 @@ using namespace llvm; +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OperandBundleDef, LLVMOperandBundleRef) + #define DEBUG_TYPE "ir" void llvm::initializeCore(PassRegistry &Registry) { @@ -2567,6 +2569,34 @@ void LLVMRemoveGlobalIFunc(LLVMValueRef IFunc) { unwrap(IFunc)->removeFromParent(); } +/*--.. Operations on operand bundles........................................--*/ + +LLVMOperandBundleRef LLVMCreateOperandBundle(const char *Tag, size_t TagLen, + LLVMValueRef *Args, + unsigned NumArgs) { + return wrap(new OperandBundleDef(std::string(Tag, TagLen), + ArrayRef(unwrap(Args), NumArgs))); +} + +void LLVMDisposeOperandBundle(LLVMOperandBundleRef Bundle) { + delete unwrap(Bundle); +} + +const char *LLVMGetOperandBundleTag(LLVMOperandBundleRef Bundle, size_t *Len) { + StringRef Str = unwrap(Bundle)->getTag(); + *Len = Str.size(); + return Str.data(); +} + +unsigned LLVMGetNumOperandBundleArgs(LLVMOperandBundleRef Bundle) { + return unwrap(Bundle)->inputs().size(); +} + +LLVMValueRef LLVMGetOperandBundleArgAtIndex(LLVMOperandBundleRef Bundle, + unsigned Index) { + return wrap(unwrap(Bundle)->inputs()[Index]); +} + /*--.. Operations on basic blocks ..........................................--*/ LLVMValueRef LLVMBasicBlockAsValue(LLVMBasicBlockRef BB) { @@ -2858,6 +2888,16 @@ LLVMTypeRef LLVMGetCalledFunctionType(LLVMValueRef Instr) { return wrap(unwrap(Instr)->getFunctionType()); } +unsigned LLVMGetNumOperandBundles(LLVMValueRef C) { + return unwrap(C)->getNumOperandBundles(); +} + +LLVMOperandBundleRef LLVMGetOperandBundleAtIndex(LLVMValueRef C, + unsigned Index) { + return wrap( + new OperandBundleDef(unwrap(C)->getOperandBundleAt(Index))); +} + /*--.. Operations on call instructions (only) ..............................--*/ LLVMBool LLVMIsTailCall(LLVMValueRef Call) { @@ -3140,6 +3180,20 @@ LLVMValueRef LLVMBuildInvoke2(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, ArrayRef(unwrap(Args), NumArgs), Name)); } +LLVMValueRef LLVMBuildInvokeWithOperandBundles( + LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMValueRef *Args, + unsigned NumArgs, LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, + LLVMOperandBundleRef *Bundles, unsigned NumBundles, const char *Name) { + SmallVector OBs; + for (auto *Bundle : ArrayRef(Bundles, NumBundles)) { + OperandBundleDef *OB = unwrap(Bundle); + OBs.push_back(*OB); + } + return wrap(unwrap(B)->CreateInvoke( + unwrap(Ty), unwrap(Fn), unwrap(Then), unwrap(Catch), + ArrayRef(unwrap(Args), NumArgs), OBs, Name)); +} + LLVMValueRef LLVMBuildLandingPad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef PersFn, unsigned NumClauses, const char *Name) { @@ -3878,6 +3932,21 @@ LLVMValueRef LLVMBuildCall2(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, ArrayRef(unwrap(Args), NumArgs), Name)); } +LLVMValueRef +LLVMBuildCallWithOperandBundles(LLVMBuilderRef B, LLVMTypeRef Ty, + LLVMValueRef Fn, LLVMValueRef *Args, + unsigned NumArgs, LLVMOperandBundleRef *Bundles, + unsigned NumBundles, const char *Name) { + FunctionType *FTy = unwrap(Ty); + SmallVector OBs; + for (auto *Bundle : ArrayRef(Bundles, NumBundles)) { + OperandBundleDef *OB = unwrap(Bundle); + OBs.push_back(*OB); + } + return wrap(unwrap(B)->CreateCall( + FTy, unwrap(Fn), ArrayRef(unwrap(Args), NumArgs), OBs, Name)); +} + LLVMValueRef LLVMBuildSelect(LLVMBuilderRef B, LLVMValueRef If, LLVMValueRef Then, LLVMValueRef Else, const char *Name) { diff --git a/llvm/lib/IR/DebugProgramInstruction.cpp b/llvm/lib/IR/DebugProgramInstruction.cpp index 6a4ee9d6101076fad65a54168b98b5835e9820dc..df45c6ea3a776fdcf63aa3152856981ca6ae3a91 100644 --- a/llvm/lib/IR/DebugProgramInstruction.cpp +++ b/llvm/lib/IR/DebugProgramInstruction.cpp @@ -35,10 +35,9 @@ DPValue::DPValue(const DPValue &DPV) DbgLoc(DPV.getDebugLoc()), Type(DPV.getType()) {} DPValue::DPValue(Metadata *Location, DILocalVariable *DV, DIExpression *Expr, - const DILocation *DI) + const DILocation *DI, LocationType Type) : DebugValueUser(Location), Variable(DV), Expression(Expr), DbgLoc(DI), - Type(LocationType::Value) { -} + Type(Type) {} void DPValue::deleteInstr() { delete this; } diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp index 3d2e203a20dac779f3377c2693df59c2c156496b..a185ca3fb8dc11ad715234e5914c8bdf78e1e729 100644 --- a/llvm/lib/IR/Type.cpp +++ b/llvm/lib/IR/Type.cpp @@ -141,16 +141,9 @@ bool Type::canLosslesslyBitCastTo(Type *Ty) const { Ty->getPrimitiveSizeInBits().getFixedValue() == 8192) return true; - // At this point we have only various mismatches of the first class types - // remaining and ptr->ptr. Just select the lossless conversions. Everything - // else is not lossless. Conservatively assume we can't losslessly convert - // between pointers with different address spaces. - if (auto *PTy = dyn_cast(this)) { - if (auto *OtherPTy = dyn_cast(Ty)) - return PTy->getAddressSpace() == OtherPTy->getAddressSpace(); - return false; - } - return false; // Other types have no identity values + // Conservatively assume we can't losslessly convert between pointers with + // different address spaces. + return false; } bool Type::isEmptyTy() const { diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index f137f0468c3c5c214f838668d8341686a919b2b8..c87f164bdd0f5c47706e2d7443ae8158fc312b06 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -5374,7 +5374,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { Check(cast(Call.getArgOperand(1))->getZExtValue() < 2, "rw argument to llvm.prefetch must be 0-1", Call); Check(cast(Call.getArgOperand(2))->getZExtValue() < 4, - "locality argument to llvm.prefetch must be 0-4", Call); + "locality argument to llvm.prefetch must be 0-3", Call); Check(cast(Call.getArgOperand(3))->getZExtValue() < 2, "cache type argument to llvm.prefetch must be 0-1", Call); break; diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp index 901a66f156663f83a2a77dd41f569644a387637a..def13044dfccc34a282c38f5ec637c6ae2a7d4ba 100644 --- a/llvm/lib/MC/MCAssembler.cpp +++ b/llvm/lib/MC/MCAssembler.cpp @@ -193,9 +193,9 @@ const MCSymbol *MCAssembler::getAtom(const MCSymbol &S) const { return S.getFragment()->getAtom(); } -bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout, - const MCFixup &Fixup, const MCFragment *DF, - MCValue &Target, uint64_t &Value, +bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout, const MCFixup &Fixup, + const MCFragment *DF, MCValue &Target, + const MCSubtargetInfo *STI, uint64_t &Value, bool &WasForced) const { ++stats::evaluateFixup; @@ -227,7 +227,7 @@ bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout, if (IsTarget) return getBackend().evaluateTargetFixup(*this, Layout, Fixup, DF, Target, - Value, WasForced); + STI, Value, WasForced); unsigned FixupFlags = getBackendPtr()->getFixupKindInfo(Fixup.getKind()).Flags; bool IsPCRel = getBackendPtr()->getFixupKindInfo(Fixup.getKind()).Flags & @@ -282,7 +282,8 @@ bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout, } // Let the backend force a relocation if needed. - if (IsResolved && getBackend().shouldForceRelocation(*this, Fixup, Target)) { + if (IsResolved && + getBackend().shouldForceRelocation(*this, Fixup, Target, STI)) { IsResolved = false; WasForced = true; } @@ -796,13 +797,13 @@ void MCAssembler::writeSectionData(raw_ostream &OS, const MCSection *Sec, std::tuple MCAssembler::handleFixup(const MCAsmLayout &Layout, MCFragment &F, - const MCFixup &Fixup) { + const MCFixup &Fixup, const MCSubtargetInfo *STI) { // Evaluate the fixup. MCValue Target; uint64_t FixedValue; bool WasForced; - bool IsResolved = evaluateFixup(Layout, Fixup, &F, Target, FixedValue, - WasForced); + bool IsResolved = + evaluateFixup(Layout, Fixup, &F, Target, STI, FixedValue, WasForced); if (!IsResolved) { // The fixup was unresolved, we need a relocation. Inform the object // writer of the relocation, and give it an opportunity to adjust the @@ -936,7 +937,7 @@ void MCAssembler::layout(MCAsmLayout &Layout) { bool IsResolved; MCValue Target; std::tie(Target, FixedValue, IsResolved) = - handleFixup(Layout, Frag, Fixup); + handleFixup(Layout, Frag, Fixup, STI); getBackend().applyFixup(*this, Fixup, Target, Contents, FixedValue, IsResolved, STI); } @@ -960,7 +961,8 @@ bool MCAssembler::fixupNeedsRelaxation(const MCFixup &Fixup, MCValue Target; uint64_t Value; bool WasForced; - bool Resolved = evaluateFixup(Layout, Fixup, DF, Target, Value, WasForced); + bool Resolved = evaluateFixup(Layout, Fixup, DF, Target, + DF->getSubtargetInfo(), Value, WasForced); if (Target.getSymA() && Target.getSymA()->getKind() == MCSymbolRefExpr::VK_X86_ABS8 && Fixup.getKind() == FK_Data_1) diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp index 653ff4e9435a526eb6d543585e4d4a7c01291c75..e541090769e9e5c7b9fc3a8b286a119b44eee4da 100644 --- a/llvm/lib/MC/MCELFStreamer.cpp +++ b/llvm/lib/MC/MCELFStreamer.cpp @@ -501,7 +501,6 @@ void MCELFStreamer::finalizeCGProfileEntry(const MCSymbolRefExpr *&SRE, SRE->getLoc()); } const MCConstantExpr *MCOffset = MCConstantExpr::create(Offset, getContext()); - MCObjectStreamer::visitUsedExpr(*SRE); if (std::optional> Err = MCObjectStreamer::emitRelocDirective( *MCOffset, "BFD_RELOC_NONE", SRE, SRE->getLoc(), diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp index 3cf7b4359cabab6a4e94b60bd946f44f366fc8db..d11ccfb5e269f7ea4b1850eb664454199e0c5acc 100644 --- a/llvm/lib/MC/MCObjectStreamer.cpp +++ b/llvm/lib/MC/MCObjectStreamer.cpp @@ -797,8 +797,9 @@ MCObjectStreamer::emitRelocDirective(const MCExpr &Offset, StringRef Name, return std::make_pair(true, std::string("unknown relocation name")); MCFixupKind Kind = *MaybeKind; - - if (Expr == nullptr) + if (Expr) + visitUsedExpr(*Expr); + else Expr = MCSymbolRefExpr::create(getContext().createTempSymbol(), getContext()); diff --git a/llvm/lib/MC/MCSchedule.cpp b/llvm/lib/MC/MCSchedule.cpp index 990a693559a777692f3a233874a01ebb829389cd..4f7125864c5a01b1b93d37ae8e731259c93ee0ee 100644 --- a/llvm/lib/MC/MCSchedule.cpp +++ b/llvm/lib/MC/MCSchedule.cpp @@ -20,8 +20,8 @@ using namespace llvm; -static_assert(std::is_pod::value, - "We shouldn't have a static constructor here"); +static_assert(std::is_trivial_v, + "MCSchedModel is required to be a trivial type"); const MCSchedModel MCSchedModel::Default = {DefaultIssueWidth, DefaultMicroOpBufferSize, DefaultLoopMicroOpBufferSize, @@ -30,7 +30,7 @@ const MCSchedModel MCSchedModel::Default = {DefaultIssueWidth, DefaultMispredictPenalty, false, true, - false /*EnableIntervals*/, + /*EnableIntervals=*/false, 0, nullptr, nullptr, diff --git a/llvm/lib/MC/MCSubtargetInfo.cpp b/llvm/lib/MC/MCSubtargetInfo.cpp index 8ee823e0377b730ea7b2d1f157113db7ce5aaacb..cf3aba17fc3d4a357801eec8f93f21d4cc913838 100644 --- a/llvm/lib/MC/MCSubtargetInfo.cpp +++ b/llvm/lib/MC/MCSubtargetInfo.cpp @@ -214,7 +214,7 @@ void MCSubtargetInfo::InitMCProcessorInfo(StringRef CPU, StringRef TuneCPU, if (!TuneCPU.empty()) CPUSchedModel = &getSchedModelForCPU(TuneCPU); else - CPUSchedModel = &MCSchedModel::GetDefaultSchedModel(); + CPUSchedModel = &MCSchedModel::Default; } void MCSubtargetInfo::setDefaultFeatures(StringRef CPU, StringRef TuneCPU, @@ -319,7 +319,7 @@ const MCSchedModel &MCSubtargetInfo::getSchedModelForCPU(StringRef CPU) const { errs() << "'" << CPU << "' is not a recognized processor for this target" << " (ignoring processor)\n"; - return MCSchedModel::GetDefaultSchedModel(); + return MCSchedModel::Default; } assert(CPUEntry->SchedModel && "Missing processor SchedModel value"); return *CPUEntry->SchedModel; diff --git a/llvm/lib/ObjCopy/CommonConfig.cpp b/llvm/lib/ObjCopy/CommonConfig.cpp index e85715d0c44cb39d9042eab4fc4204779989d528..f44e70d996b2e182384153fb9b82b6ed9d565730 100644 --- a/llvm/lib/ObjCopy/CommonConfig.cpp +++ b/llvm/lib/ObjCopy/CommonConfig.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ObjCopy/CommonConfig.h" +#include "llvm/Support/Errc.h" namespace llvm { namespace objcopy { @@ -38,6 +39,12 @@ NameOrPattern::create(StringRef Pattern, MatchStyle MS, IsPositiveMatch); } case MatchStyle::Regex: { + Regex RegEx(Pattern); + std::string Err; + if (!RegEx.isValid(Err)) + return createStringError(errc::invalid_argument, + "cannot compile regular expression \'" + + Pattern + "\': " + Err); SmallVector Data; return NameOrPattern(std::make_shared( ("^" + Pattern.ltrim('^').rtrim('$') + "$").toStringRef(Data))); diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp index 1d73a6ffa73f5f973fbd3c21678d4c53aeeddb7f..36847d1a2a42235d3ce3561ab8820ba3adab6136 100644 --- a/llvm/lib/Object/ELF.cpp +++ b/llvm/lib/Object/ELF.cpp @@ -273,6 +273,7 @@ StringRef llvm::object::getELFSectionTypeName(uint32_t Machine, unsigned Type) { break; case ELF::EM_AARCH64: switch (Type) { + STRINGIFY_ENUM_CASE(ELF, SHT_AARCH64_AUTH_RELR); STRINGIFY_ENUM_CASE(ELF, SHT_AARCH64_MEMTAG_GLOBALS_DYNAMIC); STRINGIFY_ENUM_CASE(ELF, SHT_AARCH64_MEMTAG_GLOBALS_STATIC); } diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp index 1da4ea4e3edc919c352101b6c41fcf5a7faace15..c47d4eaa309dcdf21b151a20be156c72dd60caa9 100644 --- a/llvm/lib/ObjectYAML/ELFYAML.cpp +++ b/llvm/lib/ObjectYAML/ELFYAML.cpp @@ -719,6 +719,7 @@ void ScalarEnumerationTraits::enumeration( ECase(SHT_MSP430_ATTRIBUTES); break; case ELF::EM_AARCH64: + ECase(SHT_AARCH64_AUTH_RELR); ECase(SHT_AARCH64_MEMTAG_GLOBALS_STATIC); ECase(SHT_AARCH64_MEMTAG_GLOBALS_DYNAMIC); break; diff --git a/llvm/lib/ObjectYAML/XCOFFYAML.cpp b/llvm/lib/ObjectYAML/XCOFFYAML.cpp index 44ef33501b65e43c78801e00800c89749a6402cb..398b09c72170baa95897bd0f29b0b923977803af 100644 --- a/llvm/lib/ObjectYAML/XCOFFYAML.cpp +++ b/llvm/lib/ObjectYAML/XCOFFYAML.cpp @@ -280,47 +280,60 @@ static void auxSymMapping(IO &IO, XCOFFYAML::SectAuxEntForStat &AuxSym) { IO.mapOptional("NumberOfLineNum", AuxSym.NumberOfLineNum); } +template +static void ResetAuxSym(IO &IO, + std::unique_ptr &AuxSym) { + if (!IO.outputting()) + AuxSym.reset(new AuxEntT); +} + void MappingTraits>::mapping( IO &IO, std::unique_ptr &AuxSym) { - assert(!IO.outputting() && "We don't dump aux symbols currently."); const bool Is64 = static_cast(IO.getContext())->Header.Magic == (llvm::yaml::Hex16)XCOFF::XCOFF64; + XCOFFYAML::AuxSymbolType AuxType; + if (IO.outputting()) + AuxType = AuxSym.get()->Type; IO.mapRequired("Type", AuxType); switch (AuxType) { case XCOFFYAML::AUX_EXCEPT: - if (!Is64) + if (!Is64) { IO.setError("an auxiliary symbol of type AUX_EXCEPT cannot be defined in " "XCOFF32"); - AuxSym.reset(new XCOFFYAML::ExcpetionAuxEnt()); + return; + } + ResetAuxSym(IO, AuxSym); auxSymMapping(IO, *cast(AuxSym.get())); break; case XCOFFYAML::AUX_FCN: - AuxSym.reset(new XCOFFYAML::FunctionAuxEnt()); + ResetAuxSym(IO, AuxSym); auxSymMapping(IO, *cast(AuxSym.get()), Is64); break; case XCOFFYAML::AUX_SYM: - AuxSym.reset(new XCOFFYAML::BlockAuxEnt()); + ResetAuxSym(IO, AuxSym); auxSymMapping(IO, *cast(AuxSym.get()), Is64); break; case XCOFFYAML::AUX_FILE: - AuxSym.reset(new XCOFFYAML::FileAuxEnt()); + ResetAuxSym(IO, AuxSym); auxSymMapping(IO, *cast(AuxSym.get())); break; case XCOFFYAML::AUX_CSECT: - AuxSym.reset(new XCOFFYAML::CsectAuxEnt()); + ResetAuxSym(IO, AuxSym); auxSymMapping(IO, *cast(AuxSym.get()), Is64); break; case XCOFFYAML::AUX_SECT: - AuxSym.reset(new XCOFFYAML::SectAuxEntForDWARF()); + ResetAuxSym(IO, AuxSym); auxSymMapping(IO, *cast(AuxSym.get())); break; case XCOFFYAML::AUX_STAT: - if (Is64) + if (Is64) { IO.setError( "an auxiliary symbol of type AUX_STAT cannot be defined in XCOFF64"); - AuxSym.reset(new XCOFFYAML::SectAuxEntForStat()); + return; + } + ResetAuxSym(IO, AuxSym); auxSymMapping(IO, *cast(AuxSym.get())); break; } @@ -334,8 +347,7 @@ void MappingTraits::mapping(IO &IO, XCOFFYAML::Symbol &S) { IO.mapOptional("Type", S.Type); IO.mapOptional("StorageClass", S.StorageClass); IO.mapOptional("NumberOfAuxEntries", S.NumberOfAuxEntries); - if (!IO.outputting()) - IO.mapOptional("AuxEntries", S.AuxEntries); + IO.mapOptional("AuxEntries", S.AuxEntries); } void MappingTraits::mapping(IO &IO, XCOFFYAML::StringTable &Str) { diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index a5f9b5424358ecb0511c79f20077efeb8a5e5342..f26450e94187002ed6367d8484ca1b54c961b655 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -77,6 +77,7 @@ #include "llvm/CodeGen/ExpandLargeDivRem.h" #include "llvm/CodeGen/ExpandLargeFpConvert.h" #include "llvm/CodeGen/HardwareLoops.h" +#include "llvm/CodeGen/InterleavedAccess.h" #include "llvm/CodeGen/SafeStack.h" #include "llvm/CodeGen/TypePromotion.h" #include "llvm/CodeGen/WasmEHPrepare.h" diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index e7f88680655c12be8ebf87c092b8c63fb89a82f7..5c6c391049a7b2572d649ac812ece7ee6aa7ae66 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -810,7 +810,7 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, Options.DoCounterPromotion = true; Options.UseBFIInPromotion = IsCS; Options.Atomic = AtomicCounterUpdate; - MPM.addPass(InstrProfiling(Options, IsCS)); + MPM.addPass(InstrProfilingLoweringPass(Options, IsCS)); } void PassBuilder::addPGOInstrPassesForO0( @@ -837,7 +837,7 @@ void PassBuilder::addPGOInstrPassesForO0( Options.DoCounterPromotion = false; Options.UseBFIInPromotion = IsCS; Options.Atomic = AtomicCounterUpdate; - MPM.addPass(InstrProfiling(Options, IsCS)); + MPM.addPass(InstrProfilingLoweringPass(Options, IsCS)); } static InlineParams getInlineParamsFromOptLevel(OptimizationLevel Level) { diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 7462704ec2df8e768b3cd4f89b96bc1e54c0388b..56449906eb6562c6695419e143e7205ed53a6c3b 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -77,7 +77,7 @@ MODULE_PASS("inliner-wrapper-no-mandatory-first", ModuleInlinerWrapperPass(getInlineParams(), false)) MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass()) MODULE_PASS("instrorderfile", InstrOrderFilePass()) -MODULE_PASS("instrprof", InstrProfiling()) +MODULE_PASS("instrprof", InstrProfilingLoweringPass()) MODULE_PASS("internalize", InternalizePass()) MODULE_PASS("invalidate", InvalidateAllAnalysesPass()) MODULE_PASS("iroutliner", IROutlinerPass()) @@ -302,7 +302,7 @@ FUNCTION_PASS("dot-dom-only", DomOnlyPrinter()) FUNCTION_PASS("dot-post-dom", PostDomPrinter()) FUNCTION_PASS("dot-post-dom-only", PostDomOnlyPrinter()) FUNCTION_PASS("dse", DSEPass()) -FUNCTION_PASS("dwarfehprepare", DwarfEHPreparePass(TM)) +FUNCTION_PASS("dwarf-eh-prepare", DwarfEHPreparePass(TM)) FUNCTION_PASS("expand-large-div-rem", ExpandLargeDivRemPass(TM)) FUNCTION_PASS("expand-large-fp-convert", ExpandLargeFpConvertPass(TM)) FUNCTION_PASS("fix-irreducible", FixIrreduciblePass()) @@ -318,6 +318,7 @@ FUNCTION_PASS("inject-tli-mappings", InjectTLIMappings()) FUNCTION_PASS("instcount", InstCountPass()) FUNCTION_PASS("instnamer", InstructionNamerPass()) FUNCTION_PASS("instsimplify", InstSimplifyPass()) +FUNCTION_PASS("interleaved-access", InterleavedAccessPass(TM)) FUNCTION_PASS("invalidate", InvalidateAllAnalysesPass()) FUNCTION_PASS("irce", IRCEPass()) FUNCTION_PASS("jump-threading", JumpThreadingPass()) diff --git a/llvm/lib/Support/CommandLine.cpp b/llvm/lib/Support/CommandLine.cpp index a7e0cae8b855d7c99bf0e1653e44c22a7a2c2af1..31f79972125da90021efc8131d668a5089be917e 100644 --- a/llvm/lib/Support/CommandLine.cpp +++ b/llvm/lib/Support/CommandLine.cpp @@ -2372,7 +2372,7 @@ public: if (Sub == &SubCommand::getTopLevel()) { outs() << "USAGE: " << GlobalParser->ProgramName; - if (Subs.size() > 2) + if (!Subs.empty()) outs() << " [subcommand]"; outs() << " [options]"; } else { diff --git a/llvm/lib/Support/Windows/Path.inc b/llvm/lib/Support/Windows/Path.inc index e4563fd6ed9ef78465191ef37dddbc1179d6801e..168a63bb2d969d2286793a392787457c176cbe55 100644 --- a/llvm/lib/Support/Windows/Path.inc +++ b/llvm/lib/Support/Windows/Path.inc @@ -76,7 +76,7 @@ std::error_code widenPath(const Twine &Path8, SmallVectorImpl &Path16, // If the path is a long path, mangled into forward slashes, normalize // back to backslashes here. - if (Path8Str.startswith("//?/")) + if (Path8Str.starts_with("//?/")) llvm::sys::path::native(Path8Str, path::Style::windows_backslash); if (std::error_code EC = UTF8ToUTF16(Path8Str, Path16)) @@ -96,7 +96,7 @@ std::error_code widenPath(const Twine &Path8, SmallVectorImpl &Path16, const char *const LongPathPrefix = "\\\\?\\"; if ((Path16.size() + CurPathLen) < MaxPathLen || - Path8Str.startswith(LongPathPrefix)) + Path8Str.starts_with(LongPathPrefix)) return std::error_code(); if (!IsAbsolute) { diff --git a/llvm/lib/TableGen/JSONBackend.cpp b/llvm/lib/TableGen/JSONBackend.cpp index a9dc30d7f59b6e06102324a62e6aaf94b8b77c20..2a3f522a9c0ef2f03cca1a5f349fc10acd16ce35 100644 --- a/llvm/lib/TableGen/JSONBackend.cpp +++ b/llvm/lib/TableGen/JSONBackend.cpp @@ -11,7 +11,6 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ADT/ArrayRef.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td index ff256c9a8ccdf46210bad5596aff660f40cc64d6..c600bcaab2b3ea011c8d0fc6b35c74ecd27d018e 100644 --- a/llvm/lib/Target/AArch64/AArch64.td +++ b/llvm/lib/Target/AArch64/AArch64.td @@ -1480,9 +1480,9 @@ def ProcessorFeatures { FeatureFPARMv8, FeatureFullFP16, FeatureNEON, FeatureRCPC, FeatureSPE, FeatureSSBS, FeaturePerfMon]; - list NeoverseN2 = [HasV8_5aOps, FeatureBF16, FeatureETE, + list NeoverseN2 = [HasV9_0aOps, FeatureBF16, FeatureETE, FeatureMatMulInt8, FeatureMTE, FeatureSVE2, - FeatureSVE2BitPerm, FeatureTRBE, FeatureCrypto, + FeatureSVE2BitPerm, FeatureTRBE, FeaturePerfMon]; list Neoverse512TVB = [HasV8_4aOps, FeatureBF16, FeatureCacheDeepPersist, FeatureCrypto, FeatureFPARMv8, FeatureFP16FML, diff --git a/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp b/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp index 811635faaf1bfb8e42d74955dc4a5612cd6b5a4b..ff569e3dce2e9253ee63145c354a6ccc39552c43 100644 --- a/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp +++ b/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp @@ -101,7 +101,6 @@ #include "AArch64InstrInfo.h" #include "AArch64MachineFunctionInfo.h" #include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunctionPass.h" diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp index 3748f671f12371f9059df51595c3d7c9cd777777..757471d6a905e1b8e02e4e0404d7c6bf551b5ce1 100644 --- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp +++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp @@ -38,7 +38,6 @@ #include #include #include -#include #include using namespace llvm; diff --git a/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp b/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp index 32686b25f21184b154d5fedb28121b86e9ed3857..7544786d9f6c8506c3791a4975e67bd5fc866ed9 100644 --- a/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp +++ b/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp @@ -46,7 +46,6 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/DebugCounter.h" #include "llvm/Support/raw_ostream.h" -#include #include #include diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index f36607b03e76f00d10733782a20b684663e74b5e..3882e843fb69b8cc2d91449db70cd3c64a5c9eb2 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -1651,6 +1651,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive(); IsStrictFPEnabled = true; + setMaxAtomicSizeInBitsSupported(128); } void AArch64TargetLowering::addTypeForNEON(MVT VT) { @@ -13705,11 +13706,17 @@ static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS, assert(VT.getSizeInBits() == SrcVT.getSizeInBits() && "function only supposed to emit natural comparisons"); + APInt SplatValue; + APInt SplatUndef; + unsigned SplatBitSize; + bool HasAnyUndefs; + BuildVectorSDNode *BVN = dyn_cast(RHS.getNode()); - APInt CnstBits(VT.getSizeInBits(), 0); - APInt UndefBits(VT.getSizeInBits(), 0); - bool IsCnst = BVN && resolveBuildVector(BVN, CnstBits, UndefBits); - bool IsZero = IsCnst && (CnstBits == 0); + bool IsCnst = BVN && BVN->isConstantSplat(SplatValue, SplatUndef, + SplatBitSize, HasAnyUndefs); + bool IsZero = IsCnst && SplatValue == 0; + bool IsOne = IsCnst && SplatValue == 1; + bool IsMinusOne = IsCnst && SplatValue.isAllOnes(); if (SrcVT.getVectorElementType().isFloatingPoint()) { switch (CC) { @@ -13778,6 +13785,8 @@ static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS, case AArch64CC::GT: if (IsZero) return DAG.getNode(AArch64ISD::CMGTz, dl, VT, LHS); + if (IsMinusOne) + return DAG.getNode(AArch64ISD::CMGEz, dl, VT, LHS, RHS); return DAG.getNode(AArch64ISD::CMGT, dl, VT, LHS, RHS); case AArch64CC::LE: if (IsZero) @@ -13790,6 +13799,8 @@ static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS, case AArch64CC::LT: if (IsZero) return DAG.getNode(AArch64ISD::CMLTz, dl, VT, LHS); + if (IsOne) + return DAG.getNode(AArch64ISD::CMLEz, dl, VT, LHS); return DAG.getNode(AArch64ISD::CMGT, dl, VT, RHS, LHS); case AArch64CC::HI: return DAG.getNode(AArch64ISD::CMHI, dl, VT, LHS, RHS); @@ -24899,15 +24910,21 @@ AArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const { : AtomicExpansionKind::LLSC; } -// For the real atomic operations, we have ldxr/stxr up to 128 bits, +// The "default" for integer RMW operations is to expand to an LL/SC loop. +// However, with the LSE instructions (or outline-atomics mode, which provides +// library routines in place of the LSE-instructions), we can directly emit many +// operations instead. +// +// Floating-point operations are always emitted to a cmpxchg loop, because they +// may trigger a trap which aborts an LLSC sequence. TargetLowering::AtomicExpansionKind AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { + unsigned Size = AI->getType()->getPrimitiveSizeInBits(); + assert(Size <= 128 && "AtomicExpandPass should've handled larger sizes."); + if (AI->isFloatingPointOperation()) return AtomicExpansionKind::CmpXChg; - unsigned Size = AI->getType()->getPrimitiveSizeInBits(); - if (Size > 128) return AtomicExpansionKind::None; - bool CanUseLSE128 = Subtarget->hasLSE128() && Size == 128 && (AI->getOperation() == AtomicRMWInst::Xchg || AI->getOperation() == AtomicRMWInst::Or || diff --git a/llvm/lib/Target/AArch64/AArch64InstrGISel.td b/llvm/lib/Target/AArch64/AArch64InstrGISel.td index 1711360779bf74cd64bfd4ce61b0bf01d0356aee..1c88456560d3d3e37a6c365e272745aa258bc815 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrGISel.td +++ b/llvm/lib/Target/AArch64/AArch64InstrGISel.td @@ -209,7 +209,7 @@ def G_FCMLTZ : AArch64GenericInstruction { let hasSideEffects = 0; } -def G_PREFETCH : AArch64GenericInstruction { +def G_AARCH64_PREFETCH : AArch64GenericInstruction { let OutOperandList = (outs); let InOperandList = (ins type0:$imm, ptype0:$src1); let hasSideEffects = 1; @@ -287,7 +287,7 @@ def : GINodeEquiv; def : GINodeEquiv; -def : GINodeEquiv; +def : GINodeEquiv; // These are patterns that we only use for GlobalISel via the importer. def : Pat<(f32 (fadd (vector_extract (v2f32 FPR64:$Rn), (i64 0)), diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 93b8295f4f3efc6fbff399dac18dcae3a7926b09..50cbd3672fbd0d749ae1718ae6babd10d0d57b09 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -9532,9 +9532,9 @@ AArch64InstrInfo::probedStackAlloc(MachineBasicBlock::iterator MBBI, .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)) .setMIFlags(Flags); - // STR XZR, [SP] - BuildMI(*ExitMBB, ExitMBB->end(), DL, TII->get(AArch64::STRXui)) - .addReg(AArch64::XZR) + // LDR XZR, [SP] + BuildMI(*ExitMBB, ExitMBB->end(), DL, TII->get(AArch64::LDRXui)) + .addReg(AArch64::XZR, RegState::Define) .addReg(AArch64::SP) .addImm(0) .setMIFlags(Flags); diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp index a4165d0514dcc61d333001a8f278114654239b7f..3687492c3e3e460054a0ba6749c731da4de39998 100644 --- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp +++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp @@ -14,7 +14,6 @@ #include "AArch64InstrInfo.h" #include "AArch64Subtarget.h" #include "Utils/AArch64BaseInfo.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/IndirectThunks.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index fa4c93d5f77a196f2ba900d263a34a22ae603c85..0b220069a388b629995cf6a35b7ddea3e0493612 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -291,6 +291,7 @@ public: bool isLegalMaskedGather(Type *DataType, Align Alignment) const { return isLegalMaskedGatherScatter(DataType); } + bool isLegalMaskedScatter(Type *DataType, Align Alignment) const { return isLegalMaskedGatherScatter(DataType); } diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 21a412e9360dce369ab705b0bc8568d833e238f8..a35957c34a596bb03e29a4726f6a16da686e7307 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -1127,6 +1127,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) getActionDefinitionsBuilder(G_IS_FPCLASS).lower(); + getActionDefinitionsBuilder(G_PREFETCH).custom(); + getLegacyLegalizerInfo().computeTables(); verify(*ST.getInstrInfo()); } @@ -1176,6 +1178,8 @@ bool AArch64LegalizerInfo::legalizeCustom(LegalizerHelper &Helper, return legalizeExtractVectorElt(MI, MRI, Helper); case TargetOpcode::G_DYN_STACKALLOC: return legalizeDynStackAlloc(MI, Helper); + case TargetOpcode::G_PREFETCH: + return legalizePrefetch(MI, Helper); } llvm_unreachable("expected switch to return"); @@ -1349,30 +1353,6 @@ bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, Value.setReg(ExtValueReg); return true; } - case Intrinsic::prefetch: { - MachineIRBuilder MIB(MI); - auto &AddrVal = MI.getOperand(1); - - int64_t IsWrite = MI.getOperand(2).getImm(); - int64_t Locality = MI.getOperand(3).getImm(); - int64_t IsData = MI.getOperand(4).getImm(); - - bool IsStream = Locality == 0; - if (Locality != 0) { - assert(Locality <= 3 && "Prefetch locality out-of-range"); - // The locality degree is the opposite of the cache speed. - // Put the number the other way around. - // The encoding starts at 0 for level 1 - Locality = 3 - Locality; - } - - unsigned PrfOp = - (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream; - - MIB.buildInstr(AArch64::G_PREFETCH).addImm(PrfOp).add(AddrVal); - MI.eraseFromParent(); - return true; - } case Intrinsic::aarch64_prefetch: { MachineIRBuilder MIB(MI); auto &AddrVal = MI.getOperand(1); @@ -1387,7 +1367,7 @@ bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, (Target << 1) | // Cache level bits (unsigned)IsStream; // Stream bit - MIB.buildInstr(AArch64::G_PREFETCH).addImm(PrfOp).add(AddrVal); + MIB.buildInstr(AArch64::G_AARCH64_PREFETCH).addImm(PrfOp).add(AddrVal); MI.eraseFromParent(); return true; } @@ -1986,3 +1966,28 @@ bool AArch64LegalizerInfo::legalizeDynStackAlloc( MI.eraseFromParent(); return true; } + +bool AArch64LegalizerInfo::legalizePrefetch(MachineInstr &MI, + LegalizerHelper &Helper) const { + MachineIRBuilder &MIB = Helper.MIRBuilder; + auto &AddrVal = MI.getOperand(0); + + int64_t IsWrite = MI.getOperand(1).getImm(); + int64_t Locality = MI.getOperand(2).getImm(); + int64_t IsData = MI.getOperand(3).getImm(); + + bool IsStream = Locality == 0; + if (Locality != 0) { + assert(Locality <= 3 && "Prefetch locality out-of-range"); + // The locality degree is the opposite of the cache speed. + // Put the number the other way around. + // The encoding starts at 0 for level 1 + Locality = 3 - Locality; + } + + unsigned PrfOp = (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream; + + MIB.buildInstr(AArch64::G_AARCH64_PREFETCH).addImm(PrfOp).add(AddrVal); + MI.eraseFromParent(); + return true; +} diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h index 6fd859d334cd81d42b7d969fe271af188bbd3610..19f77baa77f89c09039c312d61e017b42491e6f4 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h @@ -64,6 +64,7 @@ private: bool legalizeExtractVectorElt(MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const; bool legalizeDynStackAlloc(MachineInstr &MI, LegalizerHelper &Helper) const; + bool legalizePrefetch(MachineInstr &MI, LegalizerHelper &Helper) const; const AArch64Subtarget *ST; }; } // End llvm namespace. diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp index c7ff14c252f127d412f010cb92003bb947aed6ec..a6900b8963bb3966272e5c218f5f9bc53fb35fd1 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp @@ -100,7 +100,8 @@ public: unsigned getFixupKindContainereSizeInBytes(unsigned Kind) const; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; }; } // end anonymous namespace @@ -499,7 +500,8 @@ bool AArch64AsmBackend::writeNopData(raw_ostream &OS, uint64_t Count, bool AArch64AsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo *STI) { unsigned Kind = Fixup.getKind(); if (Kind >= FirstLiteralRelocationKind) return true; diff --git a/llvm/lib/Target/AArch64/SMEABIPass.cpp b/llvm/lib/Target/AArch64/SMEABIPass.cpp index c813d92ec85b7a7cd483f658819be9cc24fd455d..3315171798d9f1bdc755b827738eaab3d5163296 100644 --- a/llvm/lib/Target/AArch64/SMEABIPass.cpp +++ b/llvm/lib/Target/AArch64/SMEABIPass.cpp @@ -15,7 +15,6 @@ #include "AArch64.h" #include "Utils/AArch64BaseInfo.h" #include "Utils/AArch64SMEAttributes.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/IR/Constants.h" #include "llvm/IR/IRBuilder.h" diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h index 323560a46f31de267b7be5cf946ee385e8397a98..1b75607e1dc32b15cb97d57a1dff5ffe9bd79954 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.h +++ b/llvm/lib/Target/AMDGPU/AMDGPU.h @@ -12,12 +12,12 @@ #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" +#include "llvm/Support/AMDGPUAddrSpace.h" #include "llvm/Support/CodeGen.h" namespace llvm { class AMDGPUTargetMachine; -class GCNTargetMachine; class TargetMachine; // GlobalISel passes @@ -390,72 +390,6 @@ enum TargetIndex { TI_SCRATCH_RSRC_DWORD2, TI_SCRATCH_RSRC_DWORD3 }; -} - -/// OpenCL uses address spaces to differentiate between -/// various memory regions on the hardware. On the CPU -/// all of the address spaces point to the same memory, -/// however on the GPU, each address space points to -/// a separate piece of memory that is unique from other -/// memory locations. -namespace AMDGPUAS { -enum : unsigned { - // The maximum value for flat, generic, local, private, constant and region. - MAX_AMDGPU_ADDRESS = 8, - - FLAT_ADDRESS = 0, ///< Address space for flat memory. - GLOBAL_ADDRESS = 1, ///< Address space for global memory (RAT0, VTX0). - REGION_ADDRESS = 2, ///< Address space for region memory. (GDS) - - CONSTANT_ADDRESS = 4, ///< Address space for constant memory (VTX2). - LOCAL_ADDRESS = 3, ///< Address space for local memory. - PRIVATE_ADDRESS = 5, ///< Address space for private memory. - - CONSTANT_ADDRESS_32BIT = 6, ///< Address space for 32-bit constant memory. - - BUFFER_FAT_POINTER = 7, ///< Address space for 160-bit buffer fat pointers. - ///< Not used in backend. - - BUFFER_RESOURCE = 8, ///< Address space for 128-bit buffer resources. - - /// Internal address spaces. Can be freely renumbered. - STREAMOUT_REGISTER = 128, ///< Address space for GS NGG Streamout registers. - /// end Internal address spaces. - - /// Address space for direct addressable parameter memory (CONST0). - PARAM_D_ADDRESS = 6, - /// Address space for indirect addressable parameter memory (VTX1). - PARAM_I_ADDRESS = 7, - - // Do not re-order the CONSTANT_BUFFER_* enums. Several places depend on - // this order to be able to dynamically index a constant buffer, for - // example: - // - // ConstantBufferAS = CONSTANT_BUFFER_0 + CBIdx - - CONSTANT_BUFFER_0 = 8, - CONSTANT_BUFFER_1 = 9, - CONSTANT_BUFFER_2 = 10, - CONSTANT_BUFFER_3 = 11, - CONSTANT_BUFFER_4 = 12, - CONSTANT_BUFFER_5 = 13, - CONSTANT_BUFFER_6 = 14, - CONSTANT_BUFFER_7 = 15, - CONSTANT_BUFFER_8 = 16, - CONSTANT_BUFFER_9 = 17, - CONSTANT_BUFFER_10 = 18, - CONSTANT_BUFFER_11 = 19, - CONSTANT_BUFFER_12 = 20, - CONSTANT_BUFFER_13 = 21, - CONSTANT_BUFFER_14 = 22, - CONSTANT_BUFFER_15 = 23, - - // Some places use this if the address space can't be determined. - UNKNOWN_ADDRESS_SPACE = ~0u, -}; -} - -namespace AMDGPU { // FIXME: Missing constant_32bit inline bool isFlatGlobalAddrSpace(unsigned AS) { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h index 48d030d64f82170a02ab6c62aa05cea9f6e01662..5ee888d9db001cbe66cb37c0f18c302873106324 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h @@ -9,7 +9,6 @@ #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUGLOBALISELUTILS_H #define LLVM_LIB_TARGET_AMDGPU_AMDGPUGLOBALISELUTILS_H -#include "llvm/ADT/ArrayRef.h" #include "llvm/CodeGen/Register.h" #include diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h index 18a7b5d7a9633e8057f3f91605694dd6d2af24ec..6d6bd86711b13f235d50963777e5372b178e6b6d 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h @@ -30,7 +30,6 @@ class MDNode; class Module; struct SIProgramInfo; class Type; -class GCNSubtarget; namespace AMDGPU { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp index 0b2bb98738be2aaed1a57127ddd1155c695ccb72..0a17b1536040dc65f615eb735095cae76b3a36ab 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp @@ -345,13 +345,13 @@ class PipelineSolver { // return the number of edges missed. int addEdges(SmallVectorImpl &SyncPipeline, SUnit *SU, int SGID, std::vector> &AddedEdges); - // Link the pipeline as if \p SU was in the SchedGroup with ID \p SGID. It - // returns the cost (in terms of missed pipeline edges), and tracks the edges - // added in \p AddedEdges + /// Link the pipeline as if \p SU was in the SchedGroup with ID \p SGID. It + /// returns the cost (in terms of missed pipeline edges), and tracks the edges + /// added in \p AddedEdges template int linkSUnit(SUnit *SU, int SGID, std::vector> &AddedEdges, T I, T E); - // Remove the edges passed via \p AddedEdges + /// Remove the edges passed via \p AddedEdges void removeEdges(const std::vector> &AddedEdges); // Convert the passed in maps to arrays for bidirectional iterators void convertSyncMapsToArrays(); @@ -847,11 +847,11 @@ protected: const SIInstrInfo *TII; public: - // Add SchedGroups to \p Pipeline to implement this Strategy. + /// Add SchedGroups to \p SyncedSchedGroups to implement this Strategy. virtual void applyIGLPStrategy( DenseMap &SyncedInstrs, DenseMap> &SyncedSchedGroups, - bool IsPostRA) = 0; + bool IsReentry) = 0; // Returns true if this strategy should be applied to a ScheduleDAG. virtual bool shouldApplyStrategy(ScheduleDAGInstrs *DAG) = 0; @@ -870,7 +870,7 @@ public: void applyIGLPStrategy( DenseMap &SyncedInstrs, DenseMap> &SyncedSchedGroups, - bool IsPostRA) override; + bool IsReentry) override; bool shouldApplyStrategy(ScheduleDAGInstrs *DAG) override { return true; } @@ -883,7 +883,7 @@ public: void MFMASmallGemmOpt::applyIGLPStrategy( DenseMap &SyncedInstrs, DenseMap> &SyncedSchedGroups, - bool IsPostRA) { + bool IsReentry) { // Count the number of MFMA instructions. unsigned MFMACount = 0; for (const MachineInstr &I : *DAG) @@ -1045,8 +1045,8 @@ private: : InstructionRule(TII, SGID, NeedsCache) {} }; - // Whether the SU shares a V_PERM predecessor with any SU in the SchedGroup - // that is /p Distance steps away + /// Whether the SU shares a V_PERM predecessor with any SU in the SchedGroup + /// that is \p Distance steps away class SharesPredWithPrevNthGroup final : public InstructionRule { private: unsigned Distance = 1; @@ -1100,7 +1100,7 @@ public: void applyIGLPStrategy( DenseMap &SyncedInstrs, DenseMap> &SyncedSchedGroups, - bool IsPostRA) override; + bool IsReentry) override; bool shouldApplyStrategy(ScheduleDAGInstrs *DAG) override { return true; } @@ -1117,12 +1117,12 @@ static unsigned DSWWithSharedVMEMCount = 0; void MFMASmallGemmSingleWaveOpt::applyIGLPStrategy( DenseMap &SyncedInstrs, DenseMap> &SyncedSchedGroups, - bool IsPostRA) { + bool IsReentry) { unsigned MFMACount = 0; unsigned DSRCount = 0; - assert((IsPostRA || (DSWCount == 0 && DSWWithPermCount == 0 && - DSWWithSharedVMEMCount == 0)) && + assert((IsReentry || (DSWCount == 0 && DSWWithPermCount == 0 && + DSWWithSharedVMEMCount == 0)) && "DSWCounters should be zero in pre-RA scheduling!"); SmallVector DSWithPerms; for (auto &SU : DAG->SUnits) { @@ -1132,7 +1132,7 @@ void MFMASmallGemmSingleWaveOpt::applyIGLPStrategy( else if (TII->isDS(*I)) { if (I->mayLoad()) ++DSRCount; - else if (I->mayStore() && !IsPostRA) { + else if (I->mayStore() && !IsReentry) { ++DSWCount; for (auto Pred : SU.Preds) { if (Pred.getSUnit()->getInstr()->getOpcode() == @@ -1145,7 +1145,7 @@ void MFMASmallGemmSingleWaveOpt::applyIGLPStrategy( } } - if (!IsPostRA) { + if (!IsReentry) { DSWWithPermCount = DSWithPerms.size(); auto I = DSWithPerms.begin(); auto E = DSWithPerms.end(); @@ -1412,11 +1412,11 @@ public: // first created SchedGroup first. bool IsBottomUp = 1; - // Whether the mutation is being applied to post RA scheduling - bool IsPostRA = false; + // Whether or not this is a reentry into the IGroupLPDAGMutation. + bool IsReentry = false; IGroupLPDAGMutation() = default; - IGroupLPDAGMutation(bool IsPostRA) : IsPostRA(IsPostRA) {} + IGroupLPDAGMutation(bool IsReentry) : IsReentry(IsReentry) {} }; unsigned SchedGroup::NumSchedGroups = 0; @@ -1704,7 +1704,7 @@ void IGroupLPDAGMutation::initIGLPOpt(SUnit &SU) { auto S = createIGLPStrategy(StrategyID, DAG, TII); if (S->shouldApplyStrategy(DAG)) { IsBottomUp = S->IsBottomUp; - S->applyIGLPStrategy(SyncedInstrs, SyncedSchedGroups, IsPostRA); + S->applyIGLPStrategy(SyncedInstrs, SyncedSchedGroups, IsReentry); } } @@ -1712,8 +1712,13 @@ void IGroupLPDAGMutation::initIGLPOpt(SUnit &SU) { namespace llvm { -std::unique_ptr createIGroupLPDAGMutation(bool IsPostRA) { - return std::make_unique(IsPostRA); +/// \p IsReentry specifes whether or not this is a reentry into the +/// IGroupLPDAGMutation. Since there may be multiple scheduling passes on the +/// same scheduling region (e.g. pre and post-RA scheduling / multiple +/// scheduling "phases"), we can reenter this mutation framework more than once +/// for a given region. +std::unique_ptr createIGroupLPDAGMutation(bool IsReentry) { + return std::make_unique(IsReentry); } } // end namespace llvm diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h index eee2a48de396ffbabeeeb51d3d0259ff2a23b854..3ec8be4f889205184aa724c4f187d0cb177bac1d 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h @@ -14,7 +14,7 @@ namespace llvm { -std::unique_ptr createIGroupLPDAGMutation(bool IsPostRA); +std::unique_ptr createIGroupLPDAGMutation(bool IsReentry); } // namespace llvm diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index fb8d685409e42999c90d5f4a047a4e31d22ef005..a6d1da94b890780a128b9d45d353d51ada293f46 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -2334,9 +2334,8 @@ static SDValue combineBallotPattern(SDValue VCMP, bool &Negate) { // Note that ballot doesn't use SETEQ condition but its easy to support it // here for completeness, so in this case Negate is set true on return. auto VCMP_CC = cast(VCMP.getOperand(2))->get(); - auto *VCMP_CRHS = dyn_cast(VCMP.getOperand(1)); - if ((VCMP_CC == ISD::SETEQ || VCMP_CC == ISD::SETNE) && VCMP_CRHS && - VCMP_CRHS->isZero()) { + if ((VCMP_CC == ISD::SETEQ || VCMP_CC == ISD::SETNE) && + isNullConstant(VCMP.getOperand(1))) { auto Cond = VCMP.getOperand(0); if (ISD::isExtOpcode(Cond->getOpcode())) // Skip extension. @@ -2370,8 +2369,8 @@ void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) { Cond->getOperand(0)->getOpcode() == AMDGPUISD::SETCC) { SDValue VCMP = Cond->getOperand(0); auto CC = cast(Cond->getOperand(2))->get(); - auto *CRHS = dyn_cast(Cond->getOperand(1)); - if ((CC == ISD::SETEQ || CC == ISD::SETNE) && CRHS && CRHS->isZero() && + if ((CC == ISD::SETEQ || CC == ISD::SETNE) && + isNullConstant(Cond->getOperand(1)) && // TODO: make condition below an assert after fixing ballot bitwidth. VCMP.getValueType().getSizeInBits() == ST->getWavefrontSize()) { // %VCMP = i(WaveSize) AMDGPUISD::SETCC ... diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h index 25c0b4953ab7f706c182cd7179c8c00254074371..248ee26a47eb1dbb5324d7e593ae24b11a2da4bb 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h @@ -20,7 +20,6 @@ namespace llvm { class AMDGPUSubtarget; -class GCNSubtarget; class AMDGPUMachineFunction : public MachineFunctionInfo { /// A map to keep track of local memory objects and their offsets within the diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp index 3175f6358a04535b467557b62b47cab664935d4d..1f11beb71101bcbe6620b0269d438f6e2a189e8d 100644 --- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp +++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp @@ -771,7 +771,7 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size, } DecodeStatus AMDGPUDisassembler::convertEXPInst(MCInst &MI) const { - if (STI.hasFeature(AMDGPU::FeatureGFX11)) { + if (STI.hasFeature(AMDGPU::FeatureGFX11Insts)) { // The MCInst still has these fields even though they are no longer encoded // in the GFX11 instruction. insertNamedMCOperand(MI, MCOperand::createImm(0), AMDGPU::OpName::vm); @@ -782,9 +782,13 @@ DecodeStatus AMDGPUDisassembler::convertEXPInst(MCInst &MI) const { DecodeStatus AMDGPUDisassembler::convertVINTERPInst(MCInst &MI) const { if (MI.getOpcode() == AMDGPU::V_INTERP_P10_F16_F32_inreg_gfx11 || + MI.getOpcode() == AMDGPU::V_INTERP_P10_F16_F32_inreg_gfx12 || MI.getOpcode() == AMDGPU::V_INTERP_P10_RTZ_F16_F32_inreg_gfx11 || + MI.getOpcode() == AMDGPU::V_INTERP_P10_RTZ_F16_F32_inreg_gfx12 || MI.getOpcode() == AMDGPU::V_INTERP_P2_F16_F32_inreg_gfx11 || - MI.getOpcode() == AMDGPU::V_INTERP_P2_RTZ_F16_F32_inreg_gfx11) { + MI.getOpcode() == AMDGPU::V_INTERP_P2_F16_F32_inreg_gfx12 || + MI.getOpcode() == AMDGPU::V_INTERP_P2_RTZ_F16_F32_inreg_gfx11 || + MI.getOpcode() == AMDGPU::V_INTERP_P2_RTZ_F16_F32_inreg_gfx12) { // The MCInst has this field that is not directly encoded in the // instruction. insertNamedMCOperand(MI, MCOperand::createImm(0), AMDGPU::OpName::op_sel); diff --git a/llvm/lib/Target/AMDGPU/EXPInstructions.td b/llvm/lib/Target/AMDGPU/EXPInstructions.td index 14ba01f0d67c63e86aaabcacf5b69e804fad9351..ff1d661ef6fe1da779e891cfaa26ee26ddce39d7 100644 --- a/llvm/lib/Target/AMDGPU/EXPInstructions.td +++ b/llvm/lib/Target/AMDGPU/EXPInstructions.td @@ -41,8 +41,8 @@ class EXP_Real_ComprVM } // Real instruction with optional asm operand "row_en". -class EXP_Real_Row - : EXPCommon + : EXPCommon, SIMCInstr { let AsmMatchConverter = "cvtExp"; @@ -105,12 +105,12 @@ def EXP_gfx10 : EXP_Real_gfx10<0, "EXP">; def EXP_DONE_gfx10 : EXP_Real_gfx10<1, "EXP_DONE">; //===----------------------------------------------------------------------===// -// GFX11+ +// GFX11 //===----------------------------------------------------------------------===// class EXP_Real_gfx11 : EXP_Real_Row<_row, _done, pseudo, SIEncodingFamily.GFX11>, EXPe_Row { - let AssemblerPredicate = isGFX11Plus; + let AssemblerPredicate = isGFX11Only; let DecoderNamespace = "GFX11"; let row = _row; let done = _done; @@ -121,6 +121,24 @@ def EXP_DONE_gfx11 : EXP_Real_gfx11<0, 1, "EXP_DONE">; def EXP_ROW_gfx11 : EXP_Real_gfx11<1, 0, "EXP_ROW">; def EXP_ROW_DONE_gfx11 : EXP_Real_gfx11<1, 1, "EXP_ROW_DONE">; +//===----------------------------------------------------------------------===// +// GFX12+ +//===----------------------------------------------------------------------===// + +class VEXPORT_Real_gfx12 + : EXP_Real_Row<_row, _done, pseudo, SIEncodingFamily.GFX12, "export">, + EXPe_Row, MnemonicAlias<"exp", "export">, Requires<[isGFX12Plus]> { + let AssemblerPredicate = isGFX12Plus; + let DecoderNamespace = "GFX12"; + let row = _row; + let done = _done; +} + +def EXPORT_gfx12 : VEXPORT_Real_gfx12<0, 0, "EXP">; +def EXPORT_DONE_gfx12 : VEXPORT_Real_gfx12<0, 1, "EXP_DONE">; +def EXPORT_ROW_gfx12 : VEXPORT_Real_gfx12<1, 0, "EXP_ROW">; +def EXPORT_ROW_DONE_gfx12 : VEXPORT_Real_gfx12<1, 1, "EXP_ROW_DONE">; + //===----------------------------------------------------------------------===// // EXP Patterns //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp index 5ebf834377f2c843c2305c9815cf02016ff70a68..fd8f0bebd3bec3a9a2c39f721ff5f49dbc245abe 100644 --- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp +++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp @@ -274,32 +274,42 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) { if (MI.isDebugInstr()) return; - auto DecrementDef = [this](const MachineOperand &MO) { + // Kill all defs. + GCNRegPressure DefPressure, ECDefPressure; + bool HasECDefs = false; + for (const MachineOperand &MO : MI.all_defs()) { + if (!MO.getReg().isVirtual()) + continue; + Register Reg = MO.getReg(); + LaneBitmask DefMask = getDefRegMask(MO, *MRI); + + // Treat a def as fully live at the moment of definition: keep a record. + if (MO.isEarlyClobber()) { + ECDefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI); + HasECDefs = true; + } else + DefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI); + auto I = LiveRegs.find(Reg); if (I == LiveRegs.end()) - return; + continue; LaneBitmask &LiveMask = I->second; LaneBitmask PrevMask = LiveMask; - LiveMask &= ~getDefRegMask(MO, *MRI); + LiveMask &= ~DefMask; CurPressure.inc(Reg, PrevMask, LiveMask, *MRI); if (LiveMask.none()) LiveRegs.erase(I); - }; - - // Decrement non-early-clobber defs. - SmallVector EarlyClobberDefs; - for (const MachineOperand &MO : MI.all_defs()) { - if (!MO.getReg().isVirtual()) - continue; - if (!MO.isEarlyClobber()) - DecrementDef(MO); - else - EarlyClobberDefs.push_back(&MO); } - // Increment uses. + // Update MaxPressure with defs pressure. + DefPressure += CurPressure; + if (HasECDefs) + DefPressure += ECDefPressure; + MaxPressure = max(DefPressure, MaxPressure); + + // Make uses alive. SmallVector RegUses; collectVirtualRegUses(RegUses, MI, LIS, *MRI); for (const RegisterMaskPair &U : RegUses) { @@ -309,13 +319,9 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) { CurPressure.inc(U.RegUnit, PrevMask, LiveMask, *MRI); } - // Point of maximum pressure: non-early-clobber defs are decremented and uses - // are incremented. - MaxPressure = max(CurPressure, MaxPressure); - - // Now decrement early clobber defs. - for (const MachineOperand *MO : EarlyClobberDefs) - DecrementDef(*MO); + // Update MaxPressure with uses plus early-clobber defs pressure. + MaxPressure = HasECDefs ? max(CurPressure + ECDefPressure, MaxPressure) + : max(CurPressure, MaxPressure); assert(CurPressure == getRegPressure(*MRI, LiveRegs)); } diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h index e21bf10d795ba527dadfb4452bd9467b50aff7e9..4100970fe1a962055a05d8645c1d2b7782c855fe 100644 --- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h +++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h @@ -85,6 +85,18 @@ struct GCNRegPressure { return !(*this == O); } + GCNRegPressure &operator+=(const GCNRegPressure &RHS) { + for (unsigned I = 0; I < TOTAL_KINDS; ++I) + Value[I] += RHS.Value[I]; + return *this; + } + + GCNRegPressure &operator-=(const GCNRegPressure &RHS) { + for (unsigned I = 0; I < TOTAL_KINDS; ++I) + Value[I] -= RHS.Value[I]; + return *this; + } + void dump() const; private: @@ -105,6 +117,20 @@ inline GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2) { return Res; } +inline GCNRegPressure operator+(const GCNRegPressure &P1, + const GCNRegPressure &P2) { + GCNRegPressure Sum = P1; + Sum += P2; + return Sum; +} + +inline GCNRegPressure operator-(const GCNRegPressure &P1, + const GCNRegPressure &P2) { + GCNRegPressure Diff = P1; + Diff -= P2; + return Diff; +} + class GCNRPTracker { public: using LiveRegSet = DenseMap; diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp index 6c044cae0d17f5bd14903600f9bf6f6fbb01aacc..342d518f38bfc4114f4e69879c181ba3e330acb5 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp @@ -853,7 +853,9 @@ bool GCNSchedStage::initGCNRegion() { StageID != GCNSchedStageID::UnclusteredHighRPReschedule) { SavedMutations.clear(); SavedMutations.swap(DAG.Mutations); - DAG.addMutation(createIGroupLPDAGMutation(/*IsPostRA=*/false)); + bool IsInitialStage = StageID == GCNSchedStageID::OccInitialSchedule || + StageID == GCNSchedStageID::ILPInitialSchedule; + DAG.addMutation(createIGroupLPDAGMutation(/*IsReentry=*/!IsInitialStage)); } return true; @@ -1567,7 +1569,7 @@ void GCNPostScheduleDAGMILive::schedule() { if (HasIGLPInstrs) { SavedMutations.clear(); SavedMutations.swap(Mutations); - addMutation(createIGroupLPDAGMutation(/*IsPostRA=*/true)); + addMutation(createIGroupLPDAGMutation(/*IsReentry=*/true)); } ScheduleDAGMI::schedule(); diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp index e18c04e623149bb5dc3955e31d270126ca2a8ddb..f91f36ed851b7f98ed24952899bac25ed7e75933 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp @@ -53,7 +53,8 @@ public: std::optional getFixupKind(StringRef Name) const override; const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; }; } //End anonymous namespace @@ -192,7 +193,8 @@ const MCFixupKindInfo &AMDGPUAsmBackend::getFixupKindInfo( bool AMDGPUAsmBackend::shouldForceRelocation(const MCAssembler &, const MCFixup &Fixup, - const MCValue &) { + const MCValue &, + const MCSubtargetInfo *STI) { return Fixup.getKind() >= FirstLiteralRelocationKind; } diff --git a/llvm/lib/Target/AMDGPU/SIDefines.h b/llvm/lib/Target/AMDGPU/SIDefines.h index 47dc59e77dc4e78f935e54f6bc2bb64f8b0585b1..29397e109706a0dcbcb41b1682c4b7ce4a1651b2 100644 --- a/llvm/lib/Target/AMDGPU/SIDefines.h +++ b/llvm/lib/Target/AMDGPU/SIDefines.h @@ -413,8 +413,8 @@ enum Id { // Message ID, width(4) [3:0]. ID_DEALLOC_VGPRS_GFX11Plus = 3, // reused in GFX11 ID_SAVEWAVE = 4, // added in GFX8, removed in GFX11 - ID_STALL_WAVE_GEN = 5, // added in GFX9 - ID_HALT_WAVES = 6, // added in GFX9 + ID_STALL_WAVE_GEN = 5, // added in GFX9, removed in GFX12 + ID_HALT_WAVES = 6, // added in GFX9, removed in GFX12 ID_ORDERED_PS_DONE = 7, // added in GFX9, removed in GFX11 ID_EARLY_PRIM_DEALLOC = 8, // added in GFX9, removed in GFX10 ID_GS_ALLOC_REQ = 9, // added in GFX9 @@ -428,6 +428,7 @@ enum Id { // Message ID, width(4) [3:0]. ID_RTN_GET_REALTIME = 131, ID_RTN_SAVE_WAVE = 132, ID_RTN_GET_TBA = 133, + ID_RTN_GET_SE_AID_ID = 134, ID_MASK_PreGFX11_ = 0xF, ID_MASK_GFX11Plus_ = 0xFF diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp index 403efd6ffed35eaa6e0cc291058ab6b48d9fbd39..23434d2de0fc661c034ab9308c32fb29a636ed6e 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp @@ -36,14 +36,15 @@ namespace SendMsg { // Disable lint checking for this block since it makes the table unreadable. // NOLINTBEGIN +// clang-format off const CustomOperand Msg[] = { {{""}}, {{"MSG_INTERRUPT"}, ID_INTERRUPT}, {{"MSG_GS"}, ID_GS_PreGFX11, isNotGFX11Plus}, {{"MSG_GS_DONE"}, ID_GS_DONE_PreGFX11, isNotGFX11Plus}, {{"MSG_SAVEWAVE"}, ID_SAVEWAVE, isGFX8_GFX9_GFX10}, - {{"MSG_STALL_WAVE_GEN"}, ID_STALL_WAVE_GEN, isGFX9Plus}, - {{"MSG_HALT_WAVES"}, ID_HALT_WAVES, isGFX9Plus}, + {{"MSG_STALL_WAVE_GEN"}, ID_STALL_WAVE_GEN, isGFX9_GFX10_GFX11}, + {{"MSG_HALT_WAVES"}, ID_HALT_WAVES, isGFX9_GFX10_GFX11}, {{"MSG_ORDERED_PS_DONE"}, ID_ORDERED_PS_DONE, isGFX9_GFX10}, {{"MSG_EARLY_PRIM_DEALLOC"}, ID_EARLY_PRIM_DEALLOC, isGFX9_GFX10}, {{"MSG_GS_ALLOC_REQ"}, ID_GS_ALLOC_REQ, isGFX9Plus}, @@ -59,7 +60,9 @@ const CustomOperand Msg[] = { {{"MSG_RTN_GET_REALTIME"}, ID_RTN_GET_REALTIME, isGFX11Plus}, {{"MSG_RTN_SAVE_WAVE"}, ID_RTN_SAVE_WAVE, isGFX11Plus}, {{"MSG_RTN_GET_TBA"}, ID_RTN_GET_TBA, isGFX11Plus}, + {{"MSG_RTN_GET_SE_AID_ID"}, ID_RTN_GET_SE_AID_ID, isGFX12Plus}, }; +// clang-format on // NOLINTEND const int MSG_SIZE = static_cast( diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h index 7c6ed01a1cd490b1f6bfaf1f3b6ed21996f2c96f..e42b27f8e09e14a39074573da4c410f9142b42ce 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h @@ -14,12 +14,10 @@ namespace llvm { struct Align; class AAResults; class DataLayout; -class Function; class GlobalVariable; class LoadInst; class MemoryDef; class MemorySSA; -class Module; class Value; namespace AMDGPU { diff --git a/llvm/lib/Target/AMDGPU/VINTERPInstructions.td b/llvm/lib/Target/AMDGPU/VINTERPInstructions.td index 7d03150bf5b1173d8483ddb12cd395a645af10dd..fc563b7493adffd4a5a13e1486ab0d913309eb9a 100644 --- a/llvm/lib/Target/AMDGPU/VINTERPInstructions.td +++ b/llvm/lib/Target/AMDGPU/VINTERPInstructions.td @@ -10,7 +10,7 @@ // VINTERP encoding //===----------------------------------------------------------------------===// -class VINTERPe_gfx11 op, VOPProfile P> : Enc64 { +class VINTERPe : Enc64 { bits<8> vdst; bits<4> src0_modifiers; bits<9> src0; @@ -31,7 +31,6 @@ class VINTERPe_gfx11 op, VOPProfile P> : Enc64 { let Inst{13} = !if(P.HasOpSel, src2_modifiers{2}, 0); // op_sel(2) let Inst{14} = !if(P.HasOpSel, src0_modifiers{3}, 0); // op_sel(3) let Inst{15} = clamp; - let Inst{22-16} = op; let Inst{40-32} = src0; let Inst{49-41} = src1; let Inst{58-50} = src2; @@ -40,6 +39,14 @@ class VINTERPe_gfx11 op, VOPProfile P> : Enc64 { let Inst{63} = src2_modifiers{0}; // neg(2) } +class VINTERPe_gfx11 op, VOPProfile P> : VINTERPe

{ + let Inst{22-16} = op; +} + +class VINTERPe_gfx12 op, VOPProfile P> : VINTERPe

{ + let Inst{20-16} = op{4-0}; +} + //===----------------------------------------------------------------------===// // VOP3 VINTERP //===----------------------------------------------------------------------===// @@ -171,17 +178,28 @@ defm : VInterpF16Pat op> { +multiclass VINTERP_Real_gfx11 op> { + let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in { def _gfx11 : VINTERP_Real(NAME), SIEncodingFamily.GFX11>, VINTERPe_gfx11(NAME).Pfl>; } } -defm V_INTERP_P10_F32_inreg : VINTERP_Real_gfx11<0x000>; -defm V_INTERP_P2_F32_inreg : VINTERP_Real_gfx11<0x001>; -defm V_INTERP_P10_F16_F32_inreg : VINTERP_Real_gfx11<0x002>; -defm V_INTERP_P2_F16_F32_inreg : VINTERP_Real_gfx11<0x003>; -defm V_INTERP_P10_RTZ_F16_F32_inreg : VINTERP_Real_gfx11<0x004>; -defm V_INTERP_P2_RTZ_F16_F32_inreg : VINTERP_Real_gfx11<0x005>; +multiclass VINTERP_Real_gfx12 op> { + let AssemblerPredicate = isGFX12Only, DecoderNamespace = "GFX12" in { + def _gfx12 : + VINTERP_Real(NAME), SIEncodingFamily.GFX12>, + VINTERPe_gfx12(NAME).Pfl>; + } +} + +multiclass VINTERP_Real_gfx11_gfx12 op> : + VINTERP_Real_gfx11, VINTERP_Real_gfx12; + +defm V_INTERP_P10_F32_inreg : VINTERP_Real_gfx11_gfx12<0x000>; +defm V_INTERP_P2_F32_inreg : VINTERP_Real_gfx11_gfx12<0x001>; +defm V_INTERP_P10_F16_F32_inreg : VINTERP_Real_gfx11_gfx12<0x002>; +defm V_INTERP_P2_F16_F32_inreg : VINTERP_Real_gfx11_gfx12<0x003>; +defm V_INTERP_P10_RTZ_F16_F32_inreg : VINTERP_Real_gfx11_gfx12<0x004>; +defm V_INTERP_P2_RTZ_F16_F32_inreg : VINTERP_Real_gfx11_gfx12<0x005>; diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td index 114d33b077866a111bf2e28b54c67b3cd25dd0cb..a73042f2e411a7b51b4efa263c805bf6a1aeac83 100644 --- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td @@ -678,11 +678,22 @@ multiclass IMAD32_Pats { >; } +// Handle cases where amdgpu-codegenprepare-mul24 made a mul24 instead of a normal mul. +// We need to separate this because otherwise OtherPredicates would be overriden. +class IMAD32_Mul24_Pat: GCNPat < + (i64 (add (i64 (AMDGPUmul_u24 i32:$src0, i32:$src1)), i64:$src2)), + (inst $src0, $src1, $src2, 0 /* clamp */) + >; + // exclude pre-GFX9 where it was slow -let OtherPredicates = [HasNotMADIntraFwdBug], SubtargetPredicate = isGFX9Plus in +let OtherPredicates = [HasNotMADIntraFwdBug], SubtargetPredicate = isGFX9Plus in { defm : IMAD32_Pats; -let OtherPredicates = [HasMADIntraFwdBug], SubtargetPredicate = isGFX11Only in + def : IMAD32_Mul24_Pat; +} +let OtherPredicates = [HasMADIntraFwdBug], SubtargetPredicate = isGFX11Only in { defm : IMAD32_Pats; + def : IMAD32_Mul24_Pat; +} def VOP3_PERMLANE_Profile : VOP3_Profile, VOP3_OPSEL> { let InsVOP3OpSel = (ins IntOpSelMods:$src0_modifiers, VRegSrc_32:$src0, diff --git a/llvm/lib/Target/ARM/ARM.td b/llvm/lib/Target/ARM/ARM.td index bf64ecf6496447d89aff52641af24bf174aaba98..97d1444a553eb18106a8658c350bcc447bb6037e 100644 --- a/llvm/lib/Target/ARM/ARM.td +++ b/llvm/lib/Target/ARM/ARM.td @@ -1519,6 +1519,17 @@ def : ProcessorModel<"cortex-m85", CortexM85Model, [ARMv81mMainline, FeatureUseMISched, HasMVEFloatOps]>; +def : ProcessorModel<"cortex-m52", CortexM55Model, [ARMv81mMainline, + FeatureDSP, + FeatureFPARMv8_D16, + FeatureHasNoBranchPredictor, + FeaturePACBTI, + FeatureUseMISched, + FeaturePrefLoopAlign32, + FeatureHasSlowFPVMLx, + FeatureMVEVectorCostFactor1, + HasMVEFloatOps]>; + def : ProcNoItin<"cortex-a32", [ARMv8a, FeatureHWDivThumb, FeatureHWDivARM, @@ -1651,7 +1662,7 @@ def : ProcNoItin<"neoverse-n1", [ARMv82a, FeatureCRC, FeatureDotProd]>; -def : ProcNoItin<"neoverse-n2", [ARMv85a, +def : ProcNoItin<"neoverse-n2", [ARMv9a, FeatureBF16, FeatureMatMulInt8]>; diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp index 1505e921405081b647ec9a88505facc7f101eb84..922fa93226f29852ad328179d2cda57b570dd8ea 100644 --- a/llvm/lib/Target/ARM/ARMSubtarget.cpp +++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp @@ -298,6 +298,7 @@ void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) { case CortexM3: case CortexM7: case CortexR52: + case CortexM52: case CortexX1: case CortexX1C: break; diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h index eb5bf25f2a38262e9db86e5c5e6873f63c2b928c..43b4123a1b55713831fbd923f312d6d2778917be 100644 --- a/llvm/lib/Target/ARM/ARMSubtarget.h +++ b/llvm/lib/Target/ARM/ARMSubtarget.h @@ -32,6 +32,7 @@ #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/TargetParser/Triple.h" +#include #include #include @@ -71,6 +72,7 @@ protected: CortexA9, CortexM3, CortexM7, + CortexM52, CortexR4, CortexR4F, CortexR5, diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp index 41b3c6005231e81c60453e81b5abddc36046dccb..534434fef5ac800ec2cab38b0edebc0b8cb36b62 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp @@ -908,7 +908,8 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm, bool ARMAsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo *STI) { const MCSymbolRefExpr *A = Target.getSymA(); const MCSymbol *Sym = A ? &A->getSymbol() : nullptr; const unsigned FixupKind = Fixup.getKind(); diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h index 40d111b79706706a9383d486a28e67ab3ad7af28..328eed9b0ec4f6511e84507e1e9ee99024759ce6 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h @@ -36,7 +36,8 @@ public: const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; unsigned adjustFixupValue(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, uint64_t Value, diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp index 62404f7add48f6b2dd201010ecdfe4312cbc59f5..c62d17fd427aac1c9632ef204a5a4adb52784e26 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp @@ -12,9 +12,9 @@ //===----------------------------------------------------------------------===// #include "ARMUnwindOpAsm.h" +#include "llvm/ADT/bit.h" #include "llvm/Support/ARMEHABI.h" #include "llvm/Support/LEB128.h" -#include "llvm/Support/MathExtras.h" #include using namespace llvm; diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp index 31a814900ca501ccbf46bb1adaf3b6fbc42b968e..c4427948d3b89476a3bc8d836f512ad63624aa79 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp @@ -19,7 +19,6 @@ #include "llvm/MC/MCWinCOFFObjectWriter.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include using namespace llvm; diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp index c94469c8d9f3d0fc06a3e48ece631d19e8836ab1..d520880d73bbd50174beee0799544aba9ff27f81 100644 --- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp +++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp @@ -507,7 +507,8 @@ bool AVRAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count, bool AVRAsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo *STI) { switch ((unsigned)Fixup.getKind()) { default: return Fixup.getKind() >= FirstLiteralRelocationKind; diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h index 3081fe1fd58c0feb796ddce91d35e3b051d4a204..023660f0ff1408470d89736d27a090c089c9f10d 100644 --- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h +++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h @@ -60,7 +60,8 @@ public: const MCSubtargetInfo *STI) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; private: Triple::OSType OSType; diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h index 436cd62c2581381500fd957aaf8b4361f52d2541..5c77d183e1ef3d3da8f35cdcad57e700d753c9e3 100644 --- a/llvm/lib/Target/BPF/BPF.h +++ b/llvm/lib/Target/BPF/BPF.h @@ -16,7 +16,10 @@ #include "llvm/Target/TargetMachine.h" namespace llvm { +class BPFRegisterBankInfo; +class BPFSubtarget; class BPFTargetMachine; +class InstructionSelector; class PassRegistry; ModulePass *createBPFCheckAndAdjustIR(); @@ -27,6 +30,10 @@ FunctionPass *createBPFMIPeepholePass(); FunctionPass *createBPFMIPreEmitPeepholePass(); FunctionPass *createBPFMIPreEmitCheckingPass(); +InstructionSelector *createBPFInstructionSelector(const BPFTargetMachine &, + const BPFSubtarget &, + const BPFRegisterBankInfo &); + void initializeBPFCheckAndAdjustIRPass(PassRegistry&); void initializeBPFDAGToDAGISelPass(PassRegistry &); void initializeBPFMIPeepholePass(PassRegistry &); diff --git a/llvm/lib/Target/BPF/BPF.td b/llvm/lib/Target/BPF/BPF.td index 7f38fbdd8c5c10af56abddc4999cd518639e6926..dff76ca07af511b5e03feba830e66e5fd1ad9384 100644 --- a/llvm/lib/Target/BPF/BPF.td +++ b/llvm/lib/Target/BPF/BPF.td @@ -11,6 +11,7 @@ include "llvm/Target/Target.td" include "BPFRegisterInfo.td" include "BPFCallingConv.td" include "BPFInstrInfo.td" +include "GISel/BPFRegisterBanks.td" def BPFInstrInfo : InstrInfo; diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td index 5972c9d49c5151ba1f6726d18975cbc7a1d57700..7d443a34490146ab28f67d8f77a77f908b64e76f 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.td +++ b/llvm/lib/Target/BPF/BPFInstrInfo.td @@ -49,8 +49,8 @@ def BPFWrapper : SDNode<"BPFISD::Wrapper", SDT_BPFWrapper>; def BPFmemcpy : SDNode<"BPFISD::MEMCPY", SDT_BPFMEMCPY, [SDNPHasChain, SDNPInGlue, SDNPOutGlue, SDNPMayStore, SDNPMayLoad]>; -def BPFIsLittleEndian : Predicate<"CurDAG->getDataLayout().isLittleEndian()">; -def BPFIsBigEndian : Predicate<"!CurDAG->getDataLayout().isLittleEndian()">; +def BPFIsLittleEndian : Predicate<"Subtarget->isLittleEndian()">; +def BPFIsBigEndian : Predicate<"!Subtarget->isLittleEndian()">; def BPFHasALU32 : Predicate<"Subtarget->getHasAlu32()">; def BPFNoALU32 : Predicate<"!Subtarget->getHasAlu32()">; def BPFHasLdsx : Predicate<"Subtarget->hasLdsx()">; diff --git a/llvm/lib/Target/BPF/BPFSubtarget.cpp b/llvm/lib/Target/BPF/BPFSubtarget.cpp index ce02c831828e4db254c00c49216421eb629eb0b8..9a8e42f3237114a6129bffcb044ea62d641c4113 100644 --- a/llvm/lib/Target/BPF/BPFSubtarget.cpp +++ b/llvm/lib/Target/BPF/BPFSubtarget.cpp @@ -12,6 +12,10 @@ #include "BPFSubtarget.h" #include "BPF.h" +#include "BPFTargetMachine.h" +#include "GISel/BPFCallLowering.h" +#include "GISel/BPFLegalizerInfo.h" +#include "GISel/BPFRegisterBankInfo.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/TargetParser/Host.h" @@ -93,4 +97,30 @@ BPFSubtarget::BPFSubtarget(const Triple &TT, const std::string &CPU, const std::string &FS, const TargetMachine &TM) : BPFGenSubtargetInfo(TT, CPU, /*TuneCPU*/ CPU, FS), FrameLowering(initializeSubtargetDependencies(CPU, FS)), - TLInfo(TM, *this) {} + TLInfo(TM, *this) { + IsLittleEndian = TT.isLittleEndian(); + + CallLoweringInfo.reset(new BPFCallLowering(*getTargetLowering())); + Legalizer.reset(new BPFLegalizerInfo(*this)); + auto *RBI = new BPFRegisterBankInfo(*getRegisterInfo()); + RegBankInfo.reset(RBI); + + InstSelector.reset(createBPFInstructionSelector( + *static_cast(&TM), *this, *RBI)); +} + +const CallLowering *BPFSubtarget::getCallLowering() const { + return CallLoweringInfo.get(); +} + +InstructionSelector *BPFSubtarget::getInstructionSelector() const { + return InstSelector.get(); +} + +const LegalizerInfo *BPFSubtarget::getLegalizerInfo() const { + return Legalizer.get(); +} + +const RegisterBankInfo *BPFSubtarget::getRegBankInfo() const { + return RegBankInfo.get(); +} diff --git a/llvm/lib/Target/BPF/BPFSubtarget.h b/llvm/lib/Target/BPF/BPFSubtarget.h index 6e81daa4d955e0dd65742a2ae7a0fde54d5a7196..33747546eadc3bfaf5b2bc8d65bc7fb3c75643a6 100644 --- a/llvm/lib/Target/BPF/BPFSubtarget.h +++ b/llvm/lib/Target/BPF/BPFSubtarget.h @@ -16,7 +16,12 @@ #include "BPFFrameLowering.h" #include "BPFISelLowering.h" #include "BPFInstrInfo.h" +#include "BPFRegisterInfo.h" #include "BPFSelectionDAGInfo.h" +#include "llvm/CodeGen/GlobalISel/CallLowering.h" +#include "llvm/CodeGen/GlobalISel/InstructionSelector.h" +#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" +#include "llvm/CodeGen/RegisterBankInfo.h" #include "llvm/CodeGen/SelectionDAGTargetInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/DataLayout.h" @@ -43,6 +48,8 @@ protected: // unused bool isDummyMode; + bool IsLittleEndian; + // whether the cpu supports jmp ext bool HasJmpExt; @@ -59,6 +66,11 @@ protected: // whether cpu v4 insns are enabled. bool HasLdsx, HasMovsx, HasBswap, HasSdivSmod, HasGotol, HasStoreImm; + std::unique_ptr CallLoweringInfo; + std::unique_ptr InstSelector; + std::unique_ptr Legalizer; + std::unique_ptr RegBankInfo; + public: // This constructor initializes the data members to match that // of the specified triple. @@ -81,6 +93,8 @@ public: bool hasGotol() const { return HasGotol; } bool hasStoreImm() const { return HasStoreImm; } + bool isLittleEndian() const { return IsLittleEndian; } + const BPFInstrInfo *getInstrInfo() const override { return &InstrInfo; } const BPFFrameLowering *getFrameLowering() const override { return &FrameLowering; @@ -91,9 +105,14 @@ public: const BPFSelectionDAGInfo *getSelectionDAGInfo() const override { return &TSInfo; } - const TargetRegisterInfo *getRegisterInfo() const override { + const BPFRegisterInfo *getRegisterInfo() const override { return &InstrInfo.getRegisterInfo(); } + + const CallLowering *getCallLowering() const override; + InstructionSelector *getInstructionSelector() const override; + const LegalizerInfo *getLegalizerInfo() const override; + const RegisterBankInfo *getRegBankInfo() const override; }; } // End llvm namespace diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp index 65286c822c4b58a9704c1eebb1d0e7e8e309677c..ab0db576f7f72de739575d53b1d836f601e5d2c5 100644 --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -15,10 +15,15 @@ #include "BPFTargetTransformInfo.h" #include "MCTargetDesc/BPFMCAsmInfo.h" #include "TargetInfo/BPFTargetInfo.h" +#include "llvm/CodeGen/GlobalISel/IRTranslator.h" +#include "llvm/CodeGen/GlobalISel/InstructionSelect.h" +#include "llvm/CodeGen/GlobalISel/Legalizer.h" +#include "llvm/CodeGen/GlobalISel/RegBankSelect.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/PassManager.h" +#include "llvm/InitializePasses.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/Support/FormattedStream.h" @@ -40,6 +45,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeBPFTarget() { RegisterTargetMachine Z(getTheBPFTarget()); PassRegistry &PR = *PassRegistry::getPassRegistry(); + initializeGlobalISel(PR); initializeBPFCheckAndAdjustIRPass(PR); initializeBPFMIPeepholePass(PR); initializeBPFDAGToDAGISelPass(PR); @@ -90,6 +96,11 @@ public: bool addInstSelector() override; void addMachineSSAOptimization() override; void addPreEmitPass() override; + + bool addIRTranslator() override; + bool addLegalizeMachineIR() override; + bool addRegBankSelect() override; + bool addGlobalInstructionSelect() override; }; } @@ -174,3 +185,23 @@ void BPFPassConfig::addPreEmitPass() { if (!DisableMIPeephole) addPass(createBPFMIPreEmitPeepholePass()); } + +bool BPFPassConfig::addIRTranslator() { + addPass(new IRTranslator()); + return false; +} + +bool BPFPassConfig::addLegalizeMachineIR() { + addPass(new Legalizer()); + return false; +} + +bool BPFPassConfig::addRegBankSelect() { + addPass(new RegBankSelect()); + return false; +} + +bool BPFPassConfig::addGlobalInstructionSelect() { + addPass(new InstructionSelect(getOptLevel())); + return false; +} diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt index 6a96394a6aee3e5c955ebd83c1d29a09f00b097b..d88e7ade40b9a04bbd3736d8842aa3c5c57dbccf 100644 --- a/llvm/lib/Target/BPF/CMakeLists.txt +++ b/llvm/lib/Target/BPF/CMakeLists.txt @@ -11,10 +11,16 @@ tablegen(LLVM BPFGenInstrInfo.inc -gen-instr-info) tablegen(LLVM BPFGenMCCodeEmitter.inc -gen-emitter) tablegen(LLVM BPFGenRegisterInfo.inc -gen-register-info) tablegen(LLVM BPFGenSubtargetInfo.inc -gen-subtarget) +tablegen(LLVM BPFGenGlobalISel.inc -gen-global-isel) +tablegen(LLVM BPFGenRegisterBank.inc -gen-register-bank) add_public_tablegen_target(BPFCommonTableGen) add_llvm_target(BPFCodeGen + GISel/BPFCallLowering.cpp + GISel/BPFInstructionSelector.cpp + GISel/BPFRegisterBankInfo.cpp + GISel/BPFLegalizerInfo.cpp BPFAbstractMemberAccess.cpp BPFAdjustOpt.cpp BPFAsmPrinter.cpp @@ -44,6 +50,7 @@ add_llvm_target(BPFCodeGen CodeGen CodeGenTypes Core + GlobalISel IPO MC Scalar diff --git a/llvm/lib/Target/BPF/GISel/BPFCallLowering.cpp b/llvm/lib/Target/BPF/GISel/BPFCallLowering.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3829a1a3151f316cfbc4a2d2ba442a76cb865b2b --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFCallLowering.cpp @@ -0,0 +1,46 @@ +//===-- BPFCallLowering.cpp - Call lowering for GlobalISel ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file implements the lowering of LLVM calls to machine code calls for +/// GlobalISel. +/// +//===----------------------------------------------------------------------===// + +#include "BPFCallLowering.h" +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "bpf-call-lowering" + +using namespace llvm; + +BPFCallLowering::BPFCallLowering(const BPFTargetLowering &TLI) + : CallLowering(&TLI) {} + +bool BPFCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, + const Value *Val, ArrayRef VRegs, + FunctionLoweringInfo &FLI, + Register SwiftErrorVReg) const { + if (!VRegs.empty()) + return false; + MIRBuilder.buildInstr(BPF::RET); + return true; +} + +bool BPFCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, + const Function &F, + ArrayRef> VRegs, + FunctionLoweringInfo &FLI) const { + return VRegs.empty(); +} + +bool BPFCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, + CallLoweringInfo &Info) const { + return false; +} diff --git a/llvm/lib/Target/BPF/GISel/BPFCallLowering.h b/llvm/lib/Target/BPF/GISel/BPFCallLowering.h new file mode 100644 index 0000000000000000000000000000000000000000..0099d2048fe521501ed3538b4401aeafb9092975 --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFCallLowering.h @@ -0,0 +1,39 @@ +//===-- BPFCallLowering.h - Call lowering for GlobalISel --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file describes how to lower LLVM calls to machine code calls. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_BPF_GISEL_BPFCALLLOWERING_H +#define LLVM_LIB_TARGET_BPF_GISEL_BPFCALLLOWERING_H + +#include "BPFISelLowering.h" +#include "llvm/CodeGen/GlobalISel/CallLowering.h" +#include "llvm/IR/CallingConv.h" + +namespace llvm { + +class BPFTargetLowering; + +class BPFCallLowering : public CallLowering { +public: + BPFCallLowering(const BPFTargetLowering &TLI); + bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, + ArrayRef VRegs, FunctionLoweringInfo &FLI, + Register SwiftErrorVReg) const override; + bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, + ArrayRef> VRegs, + FunctionLoweringInfo &FLI) const override; + bool lowerCall(MachineIRBuilder &MIRBuilder, + CallLoweringInfo &Info) const override; +}; +} // namespace llvm + +#endif diff --git a/llvm/lib/Target/BPF/GISel/BPFInstructionSelector.cpp b/llvm/lib/Target/BPF/GISel/BPFInstructionSelector.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1effeb7a57b13b8dd90edd4e975ea59fe839d8c0 --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFInstructionSelector.cpp @@ -0,0 +1,91 @@ +//===- BPFInstructionSelector.cpp --------------------------------*- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// This file implements the targeting of the InstructionSelector class for BPF. +//===----------------------------------------------------------------------===// + +#include "BPFInstrInfo.h" +#include "BPFRegisterBankInfo.h" +#include "BPFSubtarget.h" +#include "BPFTargetMachine.h" +#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" +#include "llvm/CodeGen/GlobalISel/InstructionSelector.h" +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/IR/IntrinsicsBPF.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "bpf-gisel" + +using namespace llvm; + +namespace { + +#define GET_GLOBALISEL_PREDICATE_BITSET +#include "BPFGenGlobalISel.inc" +#undef GET_GLOBALISEL_PREDICATE_BITSET + +class BPFInstructionSelector : public InstructionSelector { +public: + BPFInstructionSelector(const BPFTargetMachine &TM, const BPFSubtarget &STI, + const BPFRegisterBankInfo &RBI); + + bool select(MachineInstr &I) override; + static const char *getName() { return DEBUG_TYPE; } + +private: + /// tblgen generated 'select' implementation that is used as the initial + /// selector for the patterns that do not require complex C++. + bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; + + const BPFInstrInfo &TII; + const BPFRegisterInfo &TRI; + const BPFRegisterBankInfo &RBI; + +#define GET_GLOBALISEL_PREDICATES_DECL +#include "BPFGenGlobalISel.inc" +#undef GET_GLOBALISEL_PREDICATES_DECL + +#define GET_GLOBALISEL_TEMPORARIES_DECL +#include "BPFGenGlobalISel.inc" +#undef GET_GLOBALISEL_TEMPORARIES_DECL +}; + +} // namespace + +#define GET_GLOBALISEL_IMPL +#include "BPFGenGlobalISel.inc" +#undef GET_GLOBALISEL_IMPL + +BPFInstructionSelector::BPFInstructionSelector(const BPFTargetMachine &TM, + const BPFSubtarget &STI, + const BPFRegisterBankInfo &RBI) + : TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI), +#define GET_GLOBALISEL_PREDICATES_INIT +#include "BPFGenGlobalISel.inc" +#undef GET_GLOBALISEL_PREDICATES_INIT +#define GET_GLOBALISEL_TEMPORARIES_INIT +#include "BPFGenGlobalISel.inc" +#undef GET_GLOBALISEL_TEMPORARIES_INIT +{ +} + +bool BPFInstructionSelector::select(MachineInstr &I) { + if (selectImpl(I, *CoverageInfo)) + return true; + return false; +} + +namespace llvm { +InstructionSelector * +createBPFInstructionSelector(const BPFTargetMachine &TM, + const BPFSubtarget &Subtarget, + const BPFRegisterBankInfo &RBI) { + return new BPFInstructionSelector(TM, Subtarget, RBI); +} +} // namespace llvm diff --git a/llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.cpp b/llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.cpp new file mode 100644 index 0000000000000000000000000000000000000000..04220c176376d13cb75f96c8956bae5e1b90fe69 --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.cpp @@ -0,0 +1,22 @@ +//===- BPFLegalizerInfo.h ----------------------------------------*- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// This file implements the targeting of the Machinelegalizer class for BPF +//===----------------------------------------------------------------------===// + +#include "BPFLegalizerInfo.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "bpf-legalinfo" + +using namespace llvm; +using namespace LegalizeActions; + +BPFLegalizerInfo::BPFLegalizerInfo(const BPFSubtarget &ST) { + getLegacyLegalizerInfo().computeTables(); +} diff --git a/llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.h b/llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.h new file mode 100644 index 0000000000000000000000000000000000000000..1704bc03144c6e748c68fc33c1541c6ee20bbfe6 --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.h @@ -0,0 +1,28 @@ +//===- BPFLegalizerInfo.h ----------------------------------------*- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// This file declares the targeting of the Machinelegalizer class for BPF +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_BPF_GISEL_BPFMACHINELEGALIZER_H +#define LLVM_LIB_TARGET_BPF_GISEL_BPFMACHINELEGALIZER_H + +#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" + +namespace llvm { + +class BPFSubtarget; + +/// This class provides the information for the BPF target legalizer for +/// GlobalISel. +class BPFLegalizerInfo : public LegalizerInfo { +public: + BPFLegalizerInfo(const BPFSubtarget &ST); +}; +} // namespace llvm +#endif diff --git a/llvm/lib/Target/BPF/GISel/BPFRegisterBankInfo.cpp b/llvm/lib/Target/BPF/GISel/BPFRegisterBankInfo.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f50e8f524a872b1b900758c0b74be726153cede3 --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFRegisterBankInfo.cpp @@ -0,0 +1,25 @@ +//===- BPFRegisterBankInfo.cpp --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// This file implements the targeting of the RegisterBankInfo class for BPF +//===----------------------------------------------------------------------===// + +#include "BPFRegisterBankInfo.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "bpf-reg-bank-info" + +#define GET_TARGET_REGBANK_IMPL +#include "BPFGenRegisterBank.inc" + +using namespace llvm; + +BPFRegisterBankInfo::BPFRegisterBankInfo(const TargetRegisterInfo &TRI) + : BPFGenRegisterBankInfo() {} diff --git a/llvm/lib/Target/BPF/GISel/BPFRegisterBankInfo.h b/llvm/lib/Target/BPF/GISel/BPFRegisterBankInfo.h new file mode 100644 index 0000000000000000000000000000000000000000..82421916ca5ed4d8fa04cf4e2fa51b443eca6cbd --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFRegisterBankInfo.h @@ -0,0 +1,39 @@ +//===-- BPFRegisterBankInfo.h -----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file declares the targeting of the RegisterBankInfo class for BPF. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_BPF_GISEL_BPFREGISTERBANKINFO_H +#define LLVM_LIB_TARGET_BPF_GISEL_BPFREGISTERBANKINFO_H + +#include "MCTargetDesc/BPFMCTargetDesc.h" +#include "llvm/CodeGen/RegisterBankInfo.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" + +#define GET_REGBANK_DECLARATIONS +#include "BPFGenRegisterBank.inc" + +namespace llvm { +class TargetRegisterInfo; + +class BPFGenRegisterBankInfo : public RegisterBankInfo { +protected: +#define GET_TARGET_REGBANK_CLASS +#include "BPFGenRegisterBank.inc" +}; + +class BPFRegisterBankInfo final : public BPFGenRegisterBankInfo { +public: + BPFRegisterBankInfo(const TargetRegisterInfo &TRI); +}; +} // namespace llvm + +#endif diff --git a/llvm/lib/Target/BPF/GISel/BPFRegisterBanks.td b/llvm/lib/Target/BPF/GISel/BPFRegisterBanks.td new file mode 100644 index 0000000000000000000000000000000000000000..af4af40a253766215fce7b2586c081307acafb28 --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFRegisterBanks.td @@ -0,0 +1,15 @@ +//===-- BPFRegisterBanks.td - Describe the BPF Banks -------*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// Define the BPF register banks used for GlobalISel. +/// +//===----------------------------------------------------------------------===// + +/// General Purpose Registers +def GPRRegBank : RegisterBank<"GPRB", [GPR]>; diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp index 15ab55f95e69b81f32b66752c1cfb1d35600a61a..c266538bec7361ee767df00d1097a179cdad3dc7 100644 --- a/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp +++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp @@ -36,15 +36,16 @@ void BPFInstPrinter::printInst(const MCInst *MI, uint64_t Address, } static void printExpr(const MCExpr *Expr, raw_ostream &O) { -#ifndef NDEBUG const MCSymbolRefExpr *SRE; if (const MCBinaryExpr *BE = dyn_cast(Expr)) SRE = dyn_cast(BE->getLHS()); else SRE = dyn_cast(Expr); - assert(SRE && "Unexpected MCExpr type."); + if (!SRE) + report_fatal_error("Unexpected MCExpr type."); +#ifndef NDEBUG MCSymbolRefExpr::VariantKind Kind = SRE->getKind(); assert(Kind == MCSymbolRefExpr::VK_None); diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h b/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h index ea30e714a5b756751b89ecce87c53b2b96b4c211..f12b79586bafe3544430210101e43dbdaeb45532 100644 --- a/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h +++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h @@ -30,8 +30,7 @@ class MCSubtargetInfo; class MCTargetOptions; class Target; -MCCodeEmitter *createBPFMCCodeEmitter(const MCInstrInfo &MCII, - MCContext &Ctx); +MCCodeEmitter *createBPFMCCodeEmitter(const MCInstrInfo &MCII, MCContext &Ctx); MCCodeEmitter *createBPFbeMCCodeEmitter(const MCInstrInfo &MCII, MCContext &Ctx); @@ -43,7 +42,7 @@ MCAsmBackend *createBPFbeAsmBackend(const Target &T, const MCSubtargetInfo &STI, const MCTargetOptions &Options); std::unique_ptr createBPFELFObjectWriter(uint8_t OSABI); -} +} // namespace llvm // Defines symbolic names for BPF registers. This defines a mapping from // register name to register number. diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp index 76f5a5fc831f94318d999cb3bd9dc1d2ef397acb..bceb41a26745058d9590784f511433e7404903de 100644 --- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp +++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp @@ -262,7 +262,8 @@ bool CSKYAsmBackend::mayNeedRelaxation(const MCInst &Inst, bool CSKYAsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo * /*STI*/) { if (Fixup.getKind() >= FirstLiteralRelocationKind) return true; switch (Fixup.getTargetKind()) { diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h index 8a2f743bdee635bad2cf8a51acc6435f3f765b43..5fa0c8c01185d03126d24da30e214ad4e33c0b8a 100644 --- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h +++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h @@ -53,7 +53,8 @@ public: const MCSubtargetInfo *STI) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; std::unique_ptr createObjectTargetWriter() const override; diff --git a/llvm/lib/Target/DirectX/DXILResource.cpp b/llvm/lib/Target/DirectX/DXILResource.cpp index 0390a3f0a558d835a3509cf6cb9c8f8dff2914a1..92306d907e05461a7d231068c5716a4d243fa18f 100644 --- a/llvm/lib/Target/DirectX/DXILResource.cpp +++ b/llvm/lib/Target/DirectX/DXILResource.cpp @@ -233,9 +233,8 @@ void ResourceBase::print(raw_ostream &OS, StringRef IDPrefix, } UAVResource::UAVResource(uint32_t I, FrontendResource R) - : ResourceBase(I, R), - Shape(static_cast(R.getResourceKind())), - GloballyCoherent(false), HasCounter(false), IsROV(false), ExtProps() { + : ResourceBase(I, R), Shape(R.getResourceKind()), GloballyCoherent(false), + HasCounter(false), IsROV(R.getIsROV()), ExtProps() { parseSourceType(R.getSourceType()); } @@ -259,8 +258,6 @@ void UAVResource::print(raw_ostream &OS) const { // information we need to remove the source type string from here (See issue: // https://github.com/llvm/llvm-project/issues/57991). void UAVResource::parseSourceType(StringRef S) { - IsROV = S.startswith("RasterizerOrdered"); - S = S.substr(S.find("<") + 1); constexpr size_t PrefixLen = StringRef("vector<").size(); diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.h b/llvm/lib/Target/Hexagon/HexagonTargetMachine.h index 4ffd0fd89de6000f59337c80027ced3a9ba9c678..dddd79ad1fcfc1d2f2f4b7c06bcb0686e1605b00 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.h +++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.h @@ -21,8 +21,6 @@ namespace llvm { -class Module; - class HexagonTargetMachine : public LLVMTargetMachine { std::unique_ptr TLOF; mutable StringMap> SubtargetMap; diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp index 76b4dc4e5afa4d2364bb11cef4bb8f456a0a1f44..f9a0ba3608e6dc40fcef048d03c31fca7ff25119 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp @@ -202,7 +202,8 @@ public: } bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override { + const MCValue &Target, + const MCSubtargetInfo *STI) override { switch(Fixup.getTargetKind()) { default: llvm_unreachable("Unknown Fixup Kind!"); diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h index b83931eb88ac24a79a3d027779b7d89bd6e18d49..42d91f559f51a65f602b274079df516f36312692 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h @@ -17,7 +17,6 @@ #include "MCTargetDesc/HexagonMCInstrInfo.h" #include "MCTargetDesc/HexagonMCTargetDesc.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/Support/SMLoc.h" #include #include diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp index 400327373ad1cde115b8a001f638dcbb6b161a93..61ec63ff6c1cafc9a4ad529947d8c2cd571becef 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp @@ -247,6 +247,7 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM, setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Legal); setOperationAction(ISD::BUILD_VECTOR, VT, Custom); + setOperationAction(ISD::SETCC, VT, Legal); setOperationAction(ISD::VSELECT, VT, Legal); } for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) { @@ -260,11 +261,19 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM, setOperationAction({ISD::SHL, ISD::SRA, ISD::SRL}, VT, Legal); setOperationAction({ISD::CTPOP, ISD::CTLZ}, VT, Legal); setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Legal); + setCondCodeAction( + {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT, + Expand); } for (MVT VT : {MVT::v4f32, MVT::v2f64}) { setOperationAction({ISD::FADD, ISD::FSUB}, VT, Legal); setOperationAction({ISD::FMUL, ISD::FDIV}, VT, Legal); setOperationAction(ISD::FMA, VT, Legal); + setOperationAction(ISD::FSQRT, VT, Legal); + setOperationAction(ISD::FNEG, VT, Legal); + setCondCodeAction({ISD::SETGE, ISD::SETGT, ISD::SETOGE, ISD::SETOGT, + ISD::SETUGE, ISD::SETUGT}, + VT, Expand); } } @@ -280,6 +289,7 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM, setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Legal); setOperationAction(ISD::BUILD_VECTOR, VT, Custom); + setOperationAction(ISD::SETCC, VT, Legal); setOperationAction(ISD::VSELECT, VT, Legal); } for (MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16, MVT::v32i8}) { @@ -293,11 +303,19 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM, setOperationAction({ISD::SHL, ISD::SRA, ISD::SRL}, VT, Legal); setOperationAction({ISD::CTPOP, ISD::CTLZ}, VT, Legal); setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Legal); + setCondCodeAction( + {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT, + Expand); } for (MVT VT : {MVT::v8f32, MVT::v4f64}) { setOperationAction({ISD::FADD, ISD::FSUB}, VT, Legal); setOperationAction({ISD::FMUL, ISD::FDIV}, VT, Legal); setOperationAction(ISD::FMA, VT, Legal); + setOperationAction(ISD::FSQRT, VT, Legal); + setOperationAction(ISD::FNEG, VT, Legal); + setCondCodeAction({ISD::SETGE, ISD::SETGT, ISD::SETOGE, ISD::SETOGT, + ISD::SETUGE, ISD::SETUGT}, + VT, Expand); } } diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h index ec1e3351ac87b9ddb5feed8ae3aecaf8ae449579..2d73a73949461b5b0d7f881e793274d59d8f8244 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h @@ -21,7 +21,6 @@ namespace llvm { class LoongArchSubtarget; -struct LoongArchRegisterInfo; namespace LoongArchISD { enum NodeType : unsigned { FIRST_NUMBER = ISD::BUILTIN_OP_END, diff --git a/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td index 0bd8db1bfdf05cb16262da83a6d560d8313bf463..ec6983d0f4871fb7f09a8ab137de5b8326f7d6f5 100644 --- a/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td +++ b/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td @@ -1092,6 +1092,13 @@ multiclass PatXr { (!cast(Inst#"_D") LASX256:$xj)>; } +multiclass PatXrF { + def : Pat<(v8f32 (OpNode (v8f32 LASX256:$xj))), + (!cast(Inst#"_S") LASX256:$xj)>; + def : Pat<(v4f64 (OpNode (v4f64 LASX256:$xj))), + (!cast(Inst#"_D") LASX256:$xj)>; +} + multiclass PatXrXr { def : Pat<(OpNode (v32i8 LASX256:$xj), (v32i8 LASX256:$xk)), (!cast(Inst#"_B") LASX256:$xj, LASX256:$xk)>; @@ -1184,6 +1191,65 @@ multiclass PatShiftXrUimm { (!cast(Inst#"_D") LASX256:$xj, uimm6:$imm)>; } +multiclass PatCCXrSimm5 { + def : Pat<(v32i8 (setcc (v32i8 LASX256:$xj), + (v32i8 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_B") LASX256:$xj, simm5:$imm)>; + def : Pat<(v16i16 (setcc (v16i16 LASX256:$xj), + (v16i16 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_H") LASX256:$xj, simm5:$imm)>; + def : Pat<(v8i32 (setcc (v8i32 LASX256:$xj), + (v8i32 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_W") LASX256:$xj, simm5:$imm)>; + def : Pat<(v4i64 (setcc (v4i64 LASX256:$xj), + (v4i64 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_D") LASX256:$xj, simm5:$imm)>; +} + +multiclass PatCCXrUimm5 { + def : Pat<(v32i8 (setcc (v32i8 LASX256:$xj), + (v32i8 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_BU") LASX256:$xj, uimm5:$imm)>; + def : Pat<(v16i16 (setcc (v16i16 LASX256:$xj), + (v16i16 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_HU") LASX256:$xj, uimm5:$imm)>; + def : Pat<(v8i32 (setcc (v8i32 LASX256:$xj), + (v8i32 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_WU") LASX256:$xj, uimm5:$imm)>; + def : Pat<(v4i64 (setcc (v4i64 LASX256:$xj), + (v4i64 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_DU") LASX256:$xj, uimm5:$imm)>; +} + +multiclass PatCCXrXr { + def : Pat<(v32i8 (setcc (v32i8 LASX256:$xj), (v32i8 LASX256:$xk), CC)), + (!cast(Inst#"_B") LASX256:$xj, LASX256:$xk)>; + def : Pat<(v16i16 (setcc (v16i16 LASX256:$xj), (v16i16 LASX256:$xk), CC)), + (!cast(Inst#"_H") LASX256:$xj, LASX256:$xk)>; + def : Pat<(v8i32 (setcc (v8i32 LASX256:$xj), (v8i32 LASX256:$xk), CC)), + (!cast(Inst#"_W") LASX256:$xj, LASX256:$xk)>; + def : Pat<(v4i64 (setcc (v4i64 LASX256:$xj), (v4i64 LASX256:$xk), CC)), + (!cast(Inst#"_D") LASX256:$xj, LASX256:$xk)>; +} + +multiclass PatCCXrXrU { + def : Pat<(v32i8 (setcc (v32i8 LASX256:$xj), (v32i8 LASX256:$xk), CC)), + (!cast(Inst#"_BU") LASX256:$xj, LASX256:$xk)>; + def : Pat<(v16i16 (setcc (v16i16 LASX256:$xj), (v16i16 LASX256:$xk), CC)), + (!cast(Inst#"_HU") LASX256:$xj, LASX256:$xk)>; + def : Pat<(v8i32 (setcc (v8i32 LASX256:$xj), (v8i32 LASX256:$xk), CC)), + (!cast(Inst#"_WU") LASX256:$xj, LASX256:$xk)>; + def : Pat<(v4i64 (setcc (v4i64 LASX256:$xj), (v4i64 LASX256:$xk), CC)), + (!cast(Inst#"_DU") LASX256:$xj, LASX256:$xk)>; +} + +multiclass PatCCXrXrF { + def : Pat<(v8i32 (setcc (v8f32 LASX256:$xj), (v8f32 LASX256:$xk), CC)), + (!cast(Inst#"_S") LASX256:$xj, LASX256:$xk)>; + def : Pat<(v4i64 (setcc (v4f64 LASX256:$xj), (v4f64 LASX256:$xk), CC)), + (!cast(Inst#"_D") LASX256:$xj, LASX256:$xk)>; +} + let Predicates = [HasExtLASX] in { // XVADD_{B/H/W/D} @@ -1389,6 +1455,83 @@ def : Pat<(fma v8f32:$xj, v8f32:$xk, v8f32:$xa), def : Pat<(fma v4f64:$xj, v4f64:$xk, v4f64:$xa), (XVFMADD_D v4f64:$xj, v4f64:$xk, v4f64:$xa)>; +// XVFMSUB_{S/D} +def : Pat<(fma v8f32:$xj, v8f32:$xk, (fneg v8f32:$xa)), + (XVFMSUB_S v8f32:$xj, v8f32:$xk, v8f32:$xa)>; +def : Pat<(fma v4f64:$xj, v4f64:$xk, (fneg v4f64:$xa)), + (XVFMSUB_D v4f64:$xj, v4f64:$xk, v4f64:$xa)>; + +// XVFNMADD_{S/D} +def : Pat<(fneg (fma v8f32:$xj, v8f32:$xk, v8f32:$xa)), + (XVFNMADD_S v8f32:$xj, v8f32:$xk, v8f32:$xa)>; +def : Pat<(fneg (fma v4f64:$xj, v4f64:$xk, v4f64:$xa)), + (XVFNMADD_D v4f64:$xj, v4f64:$xk, v4f64:$xa)>; +def : Pat<(fma_nsz (fneg v8f32:$xj), v8f32:$xk, (fneg v8f32:$xa)), + (XVFNMADD_S v8f32:$xj, v8f32:$xk, v8f32:$xa)>; +def : Pat<(fma_nsz (fneg v4f64:$xj), v4f64:$xk, (fneg v4f64:$xa)), + (XVFNMADD_D v4f64:$xj, v4f64:$xk, v4f64:$xa)>; + +// XVFNMSUB_{S/D} +def : Pat<(fneg (fma v8f32:$xj, v8f32:$xk, (fneg v8f32:$xa))), + (XVFNMSUB_S v8f32:$xj, v8f32:$xk, v8f32:$xa)>; +def : Pat<(fneg (fma v4f64:$xj, v4f64:$xk, (fneg v4f64:$xa))), + (XVFNMSUB_D v4f64:$xj, v4f64:$xk, v4f64:$xa)>; +def : Pat<(fma_nsz (fneg v8f32:$xj), v8f32:$xk, v8f32:$xa), + (XVFNMSUB_S v8f32:$xj, v8f32:$xk, v8f32:$xa)>; +def : Pat<(fma_nsz (fneg v4f64:$xj), v4f64:$xk, v4f64:$xa), + (XVFNMSUB_D v4f64:$xj, v4f64:$xk, v4f64:$xa)>; + +// XVFSQRT_{S/D} +defm : PatXrF; + +// XVRECIP_{S/D} +def : Pat<(fdiv vsplatf32_fpimm_eq_1, v8f32:$xj), + (XVFRECIP_S v8f32:$xj)>; +def : Pat<(fdiv vsplatf64_fpimm_eq_1, v4f64:$xj), + (XVFRECIP_D v4f64:$xj)>; + +// XVFRSQRT_{S/D} +def : Pat<(fdiv vsplatf32_fpimm_eq_1, (fsqrt v8f32:$xj)), + (XVFRSQRT_S v8f32:$xj)>; +def : Pat<(fdiv vsplatf64_fpimm_eq_1, (fsqrt v4f64:$xj)), + (XVFRSQRT_D v4f64:$xj)>; + +// XVSEQ[I]_{B/H/W/D} +defm : PatCCXrSimm5; +defm : PatCCXrXr; + +// XVSLE[I]_{B/H/W/D}[U] +defm : PatCCXrSimm5; +defm : PatCCXrUimm5; +defm : PatCCXrXr; +defm : PatCCXrXrU; + +// XVSLT[I]_{B/H/W/D}[U] +defm : PatCCXrSimm5; +defm : PatCCXrUimm5; +defm : PatCCXrXr; +defm : PatCCXrXrU; + +// XVFCMP.cond.{S/D} +defm : PatCCXrXrF; +defm : PatCCXrXrF; +defm : PatCCXrXrF; + +defm : PatCCXrXrF; +defm : PatCCXrXrF; +defm : PatCCXrXrF; + +defm : PatCCXrXrF; +defm : PatCCXrXrF; +defm : PatCCXrXrF; + +defm : PatCCXrXrF; +defm : PatCCXrXrF; +defm : PatCCXrXrF; + +defm : PatCCXrXrF; +defm : PatCCXrXrF; + // PseudoXVINSGR2VR_{B/H} def : Pat<(vector_insert v32i8:$xd, GRLenVT:$rj, uimm5:$imm), (PseudoXVINSGR2VR_B v32i8:$xd, GRLenVT:$rj, uimm5:$imm)>; @@ -1488,6 +1631,10 @@ foreach vt = [v32i8, v16i16, v8i32, v4i64, v8f32, v4f64] in def : Pat<(vt (vselect LASX256:$xa, LASX256:$xk, LASX256:$xj)), (XVBITSEL_V LASX256:$xj, LASX256:$xk, LASX256:$xa)>; +// fneg +def : Pat<(fneg (v8f32 LASX256:$xj)), (XVBITREVI_W LASX256:$xj, 31)>; +def : Pat<(fneg (v4f64 LASX256:$xj)), (XVBITREVI_D LASX256:$xj, 63)>; + } // Predicates = [HasExtLASX] /// Intrinsic pattern diff --git a/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td index 5800ff6f62662d4eef101d31f8618d7bd2dcaad4..e468176885d758e496db6fc6ce47cf4eb074d22a 100644 --- a/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td +++ b/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td @@ -95,6 +95,29 @@ def vsplati64_imm_eq_63 : PatFrags<(ops), [(build_vector), Imm.getBitWidth() == EltTy.getSizeInBits() && Imm == 63; }]>; +def vsplatf32_fpimm_eq_1 + : PatFrags<(ops), [(bitconvert (v4i32 (build_vector))), + (bitconvert (v8i32 (build_vector)))], [{ + APInt Imm; + EVT EltTy = N->getValueType(0).getVectorElementType(); + N = N->getOperand(0).getNode(); + + return selectVSplat(N, Imm, EltTy.getSizeInBits()) && + Imm.getBitWidth() == EltTy.getSizeInBits() && + Imm == APFloat(+1.0f).bitcastToAPInt(); +}]>; +def vsplatf64_fpimm_eq_1 + : PatFrags<(ops), [(bitconvert (v2i64 (build_vector))), + (bitconvert (v4i64 (build_vector)))], [{ + APInt Imm; + EVT EltTy = N->getValueType(0).getVectorElementType(); + N = N->getOperand(0).getNode(); + + return selectVSplat(N, Imm, EltTy.getSizeInBits()) && + Imm.getBitWidth() == EltTy.getSizeInBits() && + Imm == APFloat(+1.0).bitcastToAPInt(); +}]>; + def vsplati8imm7 : PatFrag<(ops node:$reg), (and node:$reg, vsplati8_imm_eq_7)>; def vsplati16imm15 : PatFrag<(ops node:$reg), @@ -1173,6 +1196,13 @@ multiclass PatVr { (!cast(Inst#"_D") LSX128:$vj)>; } +multiclass PatVrF { + def : Pat<(v4f32 (OpNode (v4f32 LSX128:$vj))), + (!cast(Inst#"_S") LSX128:$vj)>; + def : Pat<(v2f64 (OpNode (v2f64 LSX128:$vj))), + (!cast(Inst#"_D") LSX128:$vj)>; +} + multiclass PatVrVr { def : Pat<(OpNode (v16i8 LSX128:$vj), (v16i8 LSX128:$vk)), (!cast(Inst#"_B") LSX128:$vj, LSX128:$vk)>; @@ -1261,6 +1291,65 @@ multiclass PatShiftVrUimm { (!cast(Inst#"_D") LSX128:$vj, uimm6:$imm)>; } +multiclass PatCCVrSimm5 { + def : Pat<(v16i8 (setcc (v16i8 LSX128:$vj), + (v16i8 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_B") LSX128:$vj, simm5:$imm)>; + def : Pat<(v8i16 (setcc (v8i16 LSX128:$vj), + (v8i16 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_H") LSX128:$vj, simm5:$imm)>; + def : Pat<(v4i32 (setcc (v4i32 LSX128:$vj), + (v4i32 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_W") LSX128:$vj, simm5:$imm)>; + def : Pat<(v2i64 (setcc (v2i64 LSX128:$vj), + (v2i64 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_D") LSX128:$vj, simm5:$imm)>; +} + +multiclass PatCCVrUimm5 { + def : Pat<(v16i8 (setcc (v16i8 LSX128:$vj), + (v16i8 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_BU") LSX128:$vj, uimm5:$imm)>; + def : Pat<(v8i16 (setcc (v8i16 LSX128:$vj), + (v8i16 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_HU") LSX128:$vj, uimm5:$imm)>; + def : Pat<(v4i32 (setcc (v4i32 LSX128:$vj), + (v4i32 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_WU") LSX128:$vj, uimm5:$imm)>; + def : Pat<(v2i64 (setcc (v2i64 LSX128:$vj), + (v2i64 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_DU") LSX128:$vj, uimm5:$imm)>; +} + +multiclass PatCCVrVr { + def : Pat<(v16i8 (setcc (v16i8 LSX128:$vj), (v16i8 LSX128:$vk), CC)), + (!cast(Inst#"_B") LSX128:$vj, LSX128:$vk)>; + def : Pat<(v8i16 (setcc (v8i16 LSX128:$vj), (v8i16 LSX128:$vk), CC)), + (!cast(Inst#"_H") LSX128:$vj, LSX128:$vk)>; + def : Pat<(v4i32 (setcc (v4i32 LSX128:$vj), (v4i32 LSX128:$vk), CC)), + (!cast(Inst#"_W") LSX128:$vj, LSX128:$vk)>; + def : Pat<(v2i64 (setcc (v2i64 LSX128:$vj), (v2i64 LSX128:$vk), CC)), + (!cast(Inst#"_D") LSX128:$vj, LSX128:$vk)>; +} + +multiclass PatCCVrVrU { + def : Pat<(v16i8 (setcc (v16i8 LSX128:$vj), (v16i8 LSX128:$vk), CC)), + (!cast(Inst#"_BU") LSX128:$vj, LSX128:$vk)>; + def : Pat<(v8i16 (setcc (v8i16 LSX128:$vj), (v8i16 LSX128:$vk), CC)), + (!cast(Inst#"_HU") LSX128:$vj, LSX128:$vk)>; + def : Pat<(v4i32 (setcc (v4i32 LSX128:$vj), (v4i32 LSX128:$vk), CC)), + (!cast(Inst#"_WU") LSX128:$vj, LSX128:$vk)>; + def : Pat<(v2i64 (setcc (v2i64 LSX128:$vj), (v2i64 LSX128:$vk), CC)), + (!cast(Inst#"_DU") LSX128:$vj, LSX128:$vk)>; +} + +multiclass PatCCVrVrF { + def : Pat<(v4i32 (setcc (v4f32 LSX128:$vj), (v4f32 LSX128:$vk), CC)), + (!cast(Inst#"_S") LSX128:$vj, LSX128:$vk)>; + def : Pat<(v2i64 (setcc (v2f64 LSX128:$vj), (v2f64 LSX128:$vk), CC)), + (!cast(Inst#"_D") LSX128:$vj, LSX128:$vk)>; +} + let Predicates = [HasExtLSX] in { // VADD_{B/H/W/D} @@ -1466,6 +1555,83 @@ def : Pat<(fma v4f32:$vj, v4f32:$vk, v4f32:$va), def : Pat<(fma v2f64:$vj, v2f64:$vk, v2f64:$va), (VFMADD_D v2f64:$vj, v2f64:$vk, v2f64:$va)>; +// VFMSUB_{S/D} +def : Pat<(fma v4f32:$vj, v4f32:$vk, (fneg v4f32:$va)), + (VFMSUB_S v4f32:$vj, v4f32:$vk, v4f32:$va)>; +def : Pat<(fma v2f64:$vj, v2f64:$vk, (fneg v2f64:$va)), + (VFMSUB_D v2f64:$vj, v2f64:$vk, v2f64:$va)>; + +// VFNMADD_{S/D} +def : Pat<(fneg (fma v4f32:$vj, v4f32:$vk, v4f32:$va)), + (VFNMADD_S v4f32:$vj, v4f32:$vk, v4f32:$va)>; +def : Pat<(fneg (fma v2f64:$vj, v2f64:$vk, v2f64:$va)), + (VFNMADD_D v2f64:$vj, v2f64:$vk, v2f64:$va)>; +def : Pat<(fma_nsz (fneg v4f32:$vj), v4f32:$vk, (fneg v4f32:$va)), + (VFNMADD_S v4f32:$vj, v4f32:$vk, v4f32:$va)>; +def : Pat<(fma_nsz (fneg v2f64:$vj), v2f64:$vk, (fneg v2f64:$va)), + (VFNMADD_D v2f64:$vj, v2f64:$vk, v2f64:$va)>; + +// VFNMSUB_{S/D} +def : Pat<(fneg (fma v4f32:$vj, v4f32:$vk, (fneg v4f32:$va))), + (VFNMSUB_S v4f32:$vj, v4f32:$vk, v4f32:$va)>; +def : Pat<(fneg (fma v2f64:$vj, v2f64:$vk, (fneg v2f64:$va))), + (VFNMSUB_D v2f64:$vj, v2f64:$vk, v2f64:$va)>; +def : Pat<(fma_nsz (fneg v4f32:$vj), v4f32:$vk, v4f32:$va), + (VFNMSUB_S v4f32:$vj, v4f32:$vk, v4f32:$va)>; +def : Pat<(fma_nsz (fneg v2f64:$vj), v2f64:$vk, v2f64:$va), + (VFNMSUB_D v2f64:$vj, v2f64:$vk, v2f64:$va)>; + +// VFSQRT_{S/D} +defm : PatVrF; + +// VFRECIP_{S/D} +def : Pat<(fdiv vsplatf32_fpimm_eq_1, v4f32:$vj), + (VFRECIP_S v4f32:$vj)>; +def : Pat<(fdiv vsplatf64_fpimm_eq_1, v2f64:$vj), + (VFRECIP_D v2f64:$vj)>; + +// VFRSQRT_{S/D} +def : Pat<(fdiv vsplatf32_fpimm_eq_1, (fsqrt v4f32:$vj)), + (VFRSQRT_S v4f32:$vj)>; +def : Pat<(fdiv vsplatf64_fpimm_eq_1, (fsqrt v2f64:$vj)), + (VFRSQRT_D v2f64:$vj)>; + +// VSEQ[I]_{B/H/W/D} +defm : PatCCVrSimm5; +defm : PatCCVrVr; + +// VSLE[I]_{B/H/W/D}[U] +defm : PatCCVrSimm5; +defm : PatCCVrUimm5; +defm : PatCCVrVr; +defm : PatCCVrVrU; + +// VSLT[I]_{B/H/W/D}[U] +defm : PatCCVrSimm5; +defm : PatCCVrUimm5; +defm : PatCCVrVr; +defm : PatCCVrVrU; + +// VFCMP.cond.{S/D} +defm : PatCCVrVrF; +defm : PatCCVrVrF; +defm : PatCCVrVrF; + +defm : PatCCVrVrF; +defm : PatCCVrVrF; +defm : PatCCVrVrF; + +defm : PatCCVrVrF; +defm : PatCCVrVrF; +defm : PatCCVrVrF; + +defm : PatCCVrVrF; +defm : PatCCVrVrF; +defm : PatCCVrVrF; + +defm : PatCCVrVrF; +defm : PatCCVrVrF; + // VINSGR2VR_{B/H/W/D} def : Pat<(vector_insert v16i8:$vd, GRLenVT:$rj, uimm4:$imm), (VINSGR2VR_B v16i8:$vd, GRLenVT:$rj, uimm4:$imm)>; @@ -1572,6 +1738,10 @@ foreach vt = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in def : Pat<(vt (vselect LSX128:$va, LSX128:$vk, LSX128:$vj)), (VBITSEL_V LSX128:$vj, LSX128:$vk, LSX128:$va)>; +// fneg +def : Pat<(fneg (v4f32 LSX128:$vj)), (VBITREVI_W LSX128:$vj, 31)>; +def : Pat<(fneg (v2f64 LSX128:$vj)), (VBITREVI_D LSX128:$vj, 63)>; + } // Predicates = [HasExtLSX] /// Intrinsic pattern diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp index a35916d2ad2197d9c92f617bd62600757d16ed9e..14bcef7c7d265c8598ce3e7e48ec17b4f7371337 100644 --- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp +++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp @@ -162,12 +162,13 @@ void LoongArchAsmBackend::applyFixup(const MCAssembler &Asm, bool LoongArchAsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo *STI) { if (Fixup.getKind() >= FirstLiteralRelocationKind) return true; switch (Fixup.getTargetKind()) { default: - return STI.hasFeature(LoongArch::FeatureRelax); + return STI->hasFeature(LoongArch::FeatureRelax); case FK_Data_1: case FK_Data_2: case FK_Data_4: diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h index f840f9fa2b6a007b97d0c0db95b7cddcc9c9e2ef..d1fbf788e8a8d8601404dcbf6c17127611842b15 100644 --- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h +++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h @@ -41,7 +41,8 @@ public: const MCSubtargetInfo *STI) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value, const MCRelaxableFragment *DF, diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp index 7eca49e709a0c55995059ca8a5ecee48dfc59c84..fc95b61fd4df5bed86ef0e4f0f00115ef0e38430 100644 --- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp +++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp @@ -544,7 +544,8 @@ bool MipsAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count, bool MipsAsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo *STI) { if (Fixup.getKind() >= FirstLiteralRelocationKind) return true; const unsigned FixupKind = Fixup.getKind(); diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h index 228a0b4c407c57e3fc2fbceb933b8a793cd4bbf9..2dd68b601238533504fa5076af92c46b062652bb 100644 --- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h +++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h @@ -68,7 +68,8 @@ public: const MCSubtargetInfo *STI) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; bool isMicroMips(const MCSymbol *Sym) const override; }; // class MipsAsmBackend diff --git a/llvm/lib/Target/Mips/Mips16FrameLowering.cpp b/llvm/lib/Target/Mips/Mips16FrameLowering.cpp index 4f4e3f3f2ed749dec0d44462ccb437b0bfd9e7be..10c953bb344a823c49b4e867769d9678950e3e5d 100644 --- a/llvm/lib/Target/Mips/Mips16FrameLowering.cpp +++ b/llvm/lib/Target/Mips/Mips16FrameLowering.cpp @@ -30,7 +30,6 @@ #include "llvm/MC/MachineLocation.h" #include "llvm/Support/MathExtras.h" #include "llvm/CodeGen/TargetFrameLowering.h" -#include #include #include diff --git a/llvm/lib/Target/Mips/MipsISelLowering.h b/llvm/lib/Target/Mips/MipsISelLowering.h index 81d37ff2f065fe2ee9673523dd20384dc170af8e..c17e51f4465139fa77e5981ea30b46ab682a0991 100644 --- a/llvm/lib/Target/Mips/MipsISelLowering.h +++ b/llvm/lib/Target/Mips/MipsISelLowering.h @@ -31,7 +31,6 @@ #include "llvm/IR/Type.h" #include "llvm/Target/TargetMachine.h" #include -#include #include #include #include diff --git a/llvm/lib/Target/Mips/MipsMCInstLower.cpp b/llvm/lib/Target/Mips/MipsMCInstLower.cpp index 7b58cb90ab87a6e8d4e508a2abc1f372b718f5fd..b0642f3d1ff283ab0c413f854783e2592f2b2a41 100644 --- a/llvm/lib/Target/Mips/MipsMCInstLower.cpp +++ b/llvm/lib/Target/Mips/MipsMCInstLower.cpp @@ -21,7 +21,6 @@ #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" #include "llvm/Support/ErrorHandling.h" -#include using namespace llvm; diff --git a/llvm/lib/Target/Mips/MipsSEISelLowering.cpp b/llvm/lib/Target/Mips/MipsSEISelLowering.cpp index b84f304373f6639e1a625124703e3236df64f864..5c34067c88889ad04edd4440331ebe7c9e2c0387 100644 --- a/llvm/lib/Target/Mips/MipsSEISelLowering.cpp +++ b/llvm/lib/Target/Mips/MipsSEISelLowering.cpp @@ -15,7 +15,6 @@ #include "MipsRegisterInfo.h" #include "MipsSubtarget.h" #include "llvm/ADT/APInt.h" -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/CallingConvLower.h" diff --git a/llvm/lib/Target/NVPTX/NVPTX.td b/llvm/lib/Target/NVPTX/NVPTX.td index 02fa2a4ee81ec509028c47efe593a904a1b3cdeb..f2a4ce381b40b486f870696fae1a4f5492edf75c 100644 --- a/llvm/lib/Target/NVPTX/NVPTX.td +++ b/llvm/lib/Target/NVPTX/NVPTX.td @@ -24,23 +24,24 @@ include "NVPTXInstrInfo.td" // TableGen in NVPTXGenSubtarget.inc. //===----------------------------------------------------------------------===// -class FeatureSM: - SubtargetFeature<"sm_"# version, "SmVersion", - "" # version, - "Target SM " # version>; -def SM90a: FeatureSM<90>; +class FeatureSM: + SubtargetFeature<"sm_"# sm, "FullSmVersion", + "" # value, + "Target SM " # sm>; class FeaturePTX: SubtargetFeature<"ptx"# version, "PTXVersion", "" # version, "Use PTX version " # version>; -foreach version = [20, 21, 30, 32, 35, 37, 50, 52, 53, - 60, 61, 62, 70, 72, 75, 80, 86, 87, 89, 90] in - def SM#version: FeatureSM; +foreach sm = [20, 21, 30, 32, 35, 37, 50, 52, 53, + 60, 61, 62, 70, 72, 75, 80, 86, 87, 89, 90] in + def SM#sm: FeatureSM<""#sm, !mul(sm, 10)>; + +def SM90a: FeatureSM<"90a", 901>; foreach version = [32, 40, 41, 42, 43, 50, 60, 61, 63, 64, 65, - 70, 71, 72, 73, 74, 75, 76, 77, 78, 80, 81] in + 70, 71, 72, 73, 74, 75, 76, 77, 78, 80, 81, 82, 83] in def PTX#version: FeaturePTX; //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp index 61285c6ba98dffa2f6fe59c6dd1e0799bb410a4f..f5d8abaf847a225c931466edf8842149753f401a 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -766,6 +766,17 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM, AddPromotedToType(Op, MVT::bf16, MVT::f32); } + // sm_80 only has conversions between f32 and bf16. Custom lower all other + // bf16 conversions. + if (STI.hasBF16Math() && + (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78)) { + for (MVT VT : {MVT::i1, MVT::i16, MVT::i32, MVT::i64}) { + setOperationAction( + {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT}, + VT, Custom); + } + } + setOperationAction(ISD::FROUND, MVT::f16, Promote); setOperationAction(ISD::FROUND, MVT::v2f16, Expand); setOperationAction(ISD::FROUND, MVT::v2bf16, Expand); @@ -2580,6 +2591,37 @@ SDValue NVPTXTargetLowering::LowerFROUND64(SDValue Op, return DAG.getNode(ISD::SELECT, SL, VT, IsLarge, A, RoundedA); } +SDValue NVPTXTargetLowering::LowerINT_TO_FP(SDValue Op, + SelectionDAG &DAG) const { + assert(STI.getSmVersion() < 90 || STI.getPTXVersion() < 78); + + if (Op.getValueType() == MVT::bf16) { + SDLoc Loc(Op); + return DAG.getNode( + ISD::FP_ROUND, Loc, MVT::bf16, + DAG.getNode(Op.getOpcode(), Loc, MVT::f32, Op.getOperand(0)), + DAG.getIntPtrConstant(0, Loc)); + } + + // Everything else is considered legal. + return Op; +} + +SDValue NVPTXTargetLowering::LowerFP_TO_INT(SDValue Op, + SelectionDAG &DAG) const { + assert(STI.getSmVersion() < 90 || STI.getPTXVersion() < 78); + + if (Op.getOperand(0).getValueType() == MVT::bf16) { + SDLoc Loc(Op); + return DAG.getNode( + Op.getOpcode(), Loc, Op.getValueType(), + DAG.getNode(ISD::FP_EXTEND, Loc, MVT::f32, Op.getOperand(0))); + } + + // Everything else is considered legal. + return Op; +} + static SDValue LowerVectorArith(SDValue Op, SelectionDAG &DAG) { SDLoc DL(Op); if (Op.getValueType() != MVT::v2i16) @@ -2636,6 +2678,12 @@ NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { return LowerSelect(Op, DAG); case ISD::FROUND: return LowerFROUND(Op, DAG); + case ISD::SINT_TO_FP: + case ISD::UINT_TO_FP: + return LowerINT_TO_FP(Op, DAG); + case ISD::FP_TO_SINT: + case ISD::FP_TO_UINT: + return LowerFP_TO_INT(Op, DAG); case ISD::VAARG: return LowerVAARG(Op, DAG); case ISD::VASTART: diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h index 54e34dedc6675e89803c4f8a0b488a424b06a5ec..cd6bcb048c5fe2ae4d246ea1beff650f06a716dd 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h @@ -607,6 +607,9 @@ private: SDValue LowerFROUND32(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFROUND64(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const; SDValue LowerLOADi1(SDValue Op, SelectionDAG &DAG) const; diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td index 58ff193f243714fb90bc65b8852ce1d6120e9f91..13665985f52eba5448ecf18940a45f80f6a4d2cc 100644 --- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td +++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td @@ -561,7 +561,7 @@ multiclass F2_Support_Half { [(set Int16Regs:$dst, (OpNode (bf16 Int16Regs:$a)))]>, Requires<[hasSM<80>, hasPTX<70>]>; def bf16x2 : NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$a), - !strconcat(OpcStr, ".v2bf16 \t$dst, $a;"), + !strconcat(OpcStr, ".bf16x2 \t$dst, $a;"), [(set Int32Regs:$dst, (OpNode (v2bf16 Int32Regs:$a)))]>, Requires<[hasSM<80>, hasPTX<70>]>; def f16_ftz : NVPTXInst<(outs Int16Regs:$dst), (ins Int16Regs:$a), diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp b/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp index 7fa64af196b936be37898742e4dce0f824a6eed5..420065585b38499439f742700af682fcbf06a650 100644 --- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp @@ -36,6 +36,11 @@ NVPTXSubtarget &NVPTXSubtarget::initializeSubtargetDependencies(StringRef CPU, ParseSubtargetFeatures(TargetName, /*TuneCPU*/ TargetName, FS); + // Re-map SM version numbers, SmVersion carries the regular SMs which do + // have relative order, while FullSmVersion allows distinguishing sm_90 from + // sm_90a, which would *not* be a subset of sm_91. + SmVersion = getSmVersion(); + // Set default to PTX 6.0 (CUDA 9.0) if (PTXVersion == 0) { PTXVersion = 60; @@ -48,7 +53,7 @@ NVPTXSubtarget::NVPTXSubtarget(const Triple &TT, const std::string &CPU, const std::string &FS, const NVPTXTargetMachine &TM) : NVPTXGenSubtargetInfo(TT, CPU, /*TuneCPU*/ CPU, FS), PTXVersion(0), - SmVersion(20), TM(TM), + FullSmVersion(200), SmVersion(getSmVersion()), TM(TM), TLInfo(TM, initializeSubtargetDependencies(CPU, FS)) {} bool NVPTXSubtarget::hasImageHandles() const { diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h index 93af11c258b480e0361971da168de3cf90c5347b..3ca4c1a24c79a13e57409a8acd16ad35b3ed1ce3 100644 --- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h +++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h @@ -35,7 +35,12 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo { // PTX version x.y is represented as 10*x+y, e.g. 3.1 == 31 unsigned PTXVersion; - // SM version x.y is represented as 10*x+y, e.g. 3.1 == 31 + // Full SM version x.y is represented as 100*x+10*y+feature, e.g. 3.1 == 310 + // sm_90a == 901 + unsigned int FullSmVersion; + + // SM version x.y is represented as 10*x+y, e.g. 3.1 == 31. Derived from + // FullSmVersion. unsigned int SmVersion; const NVPTXTargetMachine &TM; @@ -80,7 +85,15 @@ public: bool allowFP16Math() const; bool hasMaskOperator() const { return PTXVersion >= 71; } bool hasNoReturn() const { return SmVersion >= 30 && PTXVersion >= 64; } - unsigned int getSmVersion() const { return SmVersion; } + unsigned int getFullSmVersion() const { return FullSmVersion; } + unsigned int getSmVersion() const { return getFullSmVersion() / 10; } + // GPUs with "a" suffix have include architecture-accelerated features that + // are supported on the specified architecture only, hence such targets do not + // follow the onion layer model. hasAAFeatures() allows distinguishing such + // GPU variants from the base GPU architecture. + // - 0 represents base GPU model, + // - non-zero value identifies particular architecture-accelerated variant. + bool hasAAFeatures() const { return getFullSmVersion() % 10; } std::string getTargetName() const { return TargetName; } // Get maximum value of required alignments among the supported data types. diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp index ed52b9a9cf27b7b9a996c05a036601225056b23d..a33f44542eb5c861eb5414441af107b95b8af133 100644 --- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp +++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp @@ -886,6 +886,26 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst, Inst = TmpInst; break; } + case PPC::PLA8: + case PPC::PLA: { + MCInst TmpInst; + TmpInst.setOpcode(Opcode == PPC::PLA ? PPC::PADDI : PPC::PADDI8); + TmpInst.addOperand(Inst.getOperand(0)); + TmpInst.addOperand(Inst.getOperand(1)); + TmpInst.addOperand(Inst.getOperand(2)); + Inst = TmpInst; + break; + } + case PPC::PLA8pc: + case PPC::PLApc: { + MCInst TmpInst; + TmpInst.setOpcode(Opcode == PPC::PLApc ? PPC::PADDIpc : PPC::PADDI8pc); + TmpInst.addOperand(Inst.getOperand(0)); + TmpInst.addOperand(MCOperand::createImm(0)); + TmpInst.addOperand(Inst.getOperand(1)); + Inst = TmpInst; + break; + } case PPC::SUBI: { MCInst TmpInst; TmpInst.setOpcode(PPC::ADDI); @@ -895,6 +915,15 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst, Inst = TmpInst; break; } + case PPC::PSUBI: { + MCInst TmpInst; + TmpInst.setOpcode(PPC::PADDI); + TmpInst.addOperand(Inst.getOperand(0)); + TmpInst.addOperand(Inst.getOperand(1)); + addNegOperand(TmpInst, Inst.getOperand(2), getContext()); + Inst = TmpInst; + break; + } case PPC::SUBIS: { MCInst TmpInst; TmpInst.setOpcode(PPC::ADDIS); diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp index 8bd27571a750a844b8fbaee6a11fc68fe5a6c515..251737ed12758208f086e685629479f8eb97ec05 100644 --- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp +++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp @@ -162,7 +162,8 @@ public: } bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override { + const MCValue &Target, + const MCSubtargetInfo *STI) override { MCFixupKind Kind = Fixup.getKind(); switch ((unsigned)Kind) { default: diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.h index 7d786ac13bb902f4bb30cac7d92f9bce519ae142..10204b184a49f92875273482913eb8f10b3d8e10 100644 --- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.h +++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.h @@ -13,7 +13,6 @@ #ifndef LLVM_LIB_TARGET_PPC_MCELFSTREAMER_PPCELFSTREAMER_H #define LLVM_LIB_TARGET_PPC_MCELFSTREAMER_PPCELFSTREAMER_H -#include "llvm/ADT/SmallVector.h" #include "llvm/MC/MCELFStreamer.h" #include diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp index a5dc0b45b13cc2fabfcd3fd8ca21702f66164f57..4716e37b34432a24d4aa86e1a1e63956042da5b9 100644 --- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp +++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp @@ -12,7 +12,6 @@ #include "PPCMCAsmInfo.h" #include "llvm/TargetParser/Triple.h" -#include using namespace llvm; diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp index 284e52c298a2d0840cf8b215ff64af5125401aab..80c37f82bf29809edfe474a4aef1d7324e14c065 100644 --- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp +++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp @@ -12,7 +12,6 @@ #include "PPCPredicates.h" #include "llvm/Support/ErrorHandling.h" -#include using namespace llvm; PPC::Predicate PPC::InvertPredicate(PPC::Predicate Opcode) { diff --git a/llvm/lib/Target/PowerPC/P10InstrResources.td b/llvm/lib/Target/PowerPC/P10InstrResources.td index 66a050955631a78b6f2d0f6b599ce3c2cc84f1c2..3bbc5a63ca7abe8bde1cdd2068ac3fa38d9b18aa 100644 --- a/llvm/lib/Target/PowerPC/P10InstrResources.td +++ b/llvm/lib/Target/PowerPC/P10InstrResources.td @@ -875,7 +875,7 @@ def : InstRW<[P10W_FX_3C, P10W_DISP_ANY], MCRXRX, MFCTR, MFCTR8, MFLR, MFLR8, - WAIT + WAIT, WAITP10 )>; // 3 Cycles ALU operations, 1 input operands @@ -1884,6 +1884,7 @@ def : InstRW<[P10W_ST_3C, P10W_DISP_EVEN, P10W_DISP_ANY, P10ST_Read, P10ST_Read, def : InstRW<[P10W_ST_3C, P10W_DISP_EVEN, P10W_FX_3C, P10W_DISP_ANY], (instrs ISYNC, + SYNCP10, SYNC )>; diff --git a/llvm/lib/Target/PowerPC/PPC.h b/llvm/lib/Target/PowerPC/PPC.h index 086b61a93a43d734fd6cb54bb25b03da8344d1e2..3d9ea5608193859f28cb775b69677a9347c4842b 100644 --- a/llvm/lib/Target/PowerPC/PPC.h +++ b/llvm/lib/Target/PowerPC/PPC.h @@ -102,79 +102,99 @@ class ModulePass; // PPC Specific MachineOperand flags. MO_NO_FLAG, + /// On PPC, the 12 bits are not enough for all target operand flags. + /// Treat all PPC target flags as direct flags. To define new flag that is + /// combination of other flags, add new enum entry instead of combining + /// existing flags. See example MO_GOT_TPREL_PCREL_FLAG. + /// On a symbol operand "FOO", this indicates that the reference is actually /// to "FOO@plt". This is used for calls and jumps to external functions /// and for PIC calls on 32-bit ELF systems. - MO_PLT = 1, + MO_PLT, /// MO_PIC_FLAG - If this bit is set, the symbol reference is relative to /// the function's picbase, e.g. lo16(symbol-picbase). - MO_PIC_FLAG = 2, + MO_PIC_FLAG, /// MO_PCREL_FLAG - If this bit is set, the symbol reference is relative to /// the current instruction address(pc), e.g., var@pcrel. Fixup is VK_PCREL. - MO_PCREL_FLAG = 4, + MO_PCREL_FLAG, /// MO_GOT_FLAG - If this bit is set the symbol reference is to be computed /// via the GOT. For example when combined with the MO_PCREL_FLAG it should /// produce the relocation @got@pcrel. Fixup is VK_PPC_GOT_PCREL. - MO_GOT_FLAG = 8, + MO_GOT_FLAG, - // MO_PCREL_OPT_FLAG - If this bit is set the operand is part of a - // PC Relative linker optimization. - MO_PCREL_OPT_FLAG = 16, + /// MO_PCREL_OPT_FLAG - If this bit is set the operand is part of a + /// PC Relative linker optimization. + MO_PCREL_OPT_FLAG, /// MO_TLSGD_FLAG - If this bit is set the symbol reference is relative to /// TLS General Dynamic model for Linux and the variable offset of TLS /// General Dynamic model for AIX. - MO_TLSGD_FLAG = 32, + MO_TLSGD_FLAG, /// MO_TPREL_FLAG - If this bit is set, the symbol reference is relative to /// the thread pointer and the symbol can be used for the TLS Initial Exec /// and Local Exec models. - MO_TPREL_FLAG = 64, + MO_TPREL_FLAG, /// MO_TLSLD_FLAG - If this bit is set the symbol reference is relative to /// TLS Local Dynamic model. - MO_TLSLD_FLAG = 128, + MO_TLSLD_FLAG, /// MO_TLSGDM_FLAG - If this bit is set the symbol reference is relative /// to the region handle of TLS General Dynamic model for AIX. - MO_TLSGDM_FLAG = 256, + MO_TLSGDM_FLAG, /// MO_GOT_TLSGD_PCREL_FLAG - A combintaion of flags, if these bits are set /// they should produce the relocation @got@tlsgd@pcrel. /// Fix up is VK_PPC_GOT_TLSGD_PCREL - MO_GOT_TLSGD_PCREL_FLAG = MO_PCREL_FLAG | MO_GOT_FLAG | MO_TLSGD_FLAG, + /// MO_GOT_TLSGD_PCREL_FLAG = MO_PCREL_FLAG | MO_GOT_FLAG | MO_TLSGD_FLAG, + MO_GOT_TLSGD_PCREL_FLAG, /// MO_GOT_TLSLD_PCREL_FLAG - A combintaion of flags, if these bits are set /// they should produce the relocation @got@tlsld@pcrel. /// Fix up is VK_PPC_GOT_TLSLD_PCREL - MO_GOT_TLSLD_PCREL_FLAG = MO_PCREL_FLAG | MO_GOT_FLAG | MO_TLSLD_FLAG, + /// MO_GOT_TLSLD_PCREL_FLAG = MO_PCREL_FLAG | MO_GOT_FLAG | MO_TLSLD_FLAG, + MO_GOT_TLSLD_PCREL_FLAG, /// MO_GOT_TPREL_PCREL_FLAG - A combintaion of flags, if these bits are set /// they should produce the relocation @got@tprel@pcrel. /// Fix up is VK_PPC_GOT_TPREL_PCREL - MO_GOT_TPREL_PCREL_FLAG = MO_GOT_FLAG | MO_TPREL_FLAG | MO_PCREL_FLAG, - - /// The next are not flags but distinct values. - MO_ACCESS_MASK = 0xf00, + /// MO_GOT_TPREL_PCREL_FLAG = MO_GOT_FLAG | MO_TPREL_FLAG | MO_PCREL_FLAG, + MO_GOT_TPREL_PCREL_FLAG, /// MO_LO, MO_HA - lo16(symbol) and ha16(symbol) - MO_LO = 1 << 8, - MO_HA = 2 << 8, + MO_LO, + MO_HA, - MO_TPREL_LO = 4 << 8, - MO_TPREL_HA = 3 << 8, + MO_TPREL_LO, + MO_TPREL_HA, /// These values identify relocations on immediates folded /// into memory operations. - MO_DTPREL_LO = 5 << 8, - MO_TLSLD_LO = 6 << 8, - MO_TOC_LO = 7 << 8, + MO_DTPREL_LO, + MO_TLSLD_LO, + MO_TOC_LO, + + /// Symbol for VK_PPC_TLS fixup attached to an ADD instruction + MO_TLS, + + /// MO_PIC_HA_FLAG = MO_PIC_FLAG | MO_HA + MO_PIC_HA_FLAG, + + /// MO_PIC_LO_FLAG = MO_PIC_FLAG | MO_LO + MO_PIC_LO_FLAG, + + /// MO_TPREL_PCREL_FLAG = MO_PCREL_FLAG | MO_TPREL_FLAG + MO_TPREL_PCREL_FLAG, + + /// MO_TPREL_PCREL_FLAG = MO_PCREL_FLAG | MO_TLS + MO_TLS_PCREL_FLAG, - // Symbol for VK_PPC_TLS fixup attached to an ADD instruction - MO_TLS = 8 << 8 + /// MO_GOT_PCREL_FLAG = MO_PCREL_FLAG | MO_GOT_FLAG + MO_GOT_PCREL_FLAG, }; } // end namespace PPCII diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp index 4f15ba497d84c45b87ca7645bb13d0ce000490bb..4b551bc51c4f0e1b23bdeb9a1ae6eb06e169673d 100644 --- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp +++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp @@ -715,25 +715,11 @@ static MCSymbol *getMCSymbolForTOCPseudoMO(const MachineOperand &MO, } } -static bool hasTLSFlag(const MachineOperand &MO) { - unsigned Flags = MO.getTargetFlags(); - if (Flags & PPCII::MO_TLSGD_FLAG || Flags & PPCII::MO_TPREL_FLAG || - Flags & PPCII::MO_TLSLD_FLAG || Flags & PPCII::MO_TLSGDM_FLAG) - return true; - - if (Flags == PPCII::MO_TPREL_LO || Flags == PPCII::MO_TPREL_HA || - Flags == PPCII::MO_DTPREL_LO || Flags == PPCII::MO_TLSLD_LO || - Flags == PPCII::MO_TLS) - return true; - - return false; -} - static PPCAsmPrinter::TOCEntryType getTOCEntryTypeForMO(const MachineOperand &MO) { // Use the target flags to determine if this MO is Thread Local. // If we don't do this it comes out as Global. - if (hasTLSFlag(MO)) + if (PPCInstrInfo::hasTLSFlag(MO.getTargetFlags())) return PPCAsmPrinter::TOCType_ThreadLocal; switch (MO.getType()) { @@ -830,7 +816,10 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { // For TLS initial-exec and local-exec accesses on AIX, we have one TOC // entry for the symbol (with the variable offset), which is differentiated // by MO_TPREL_FLAG. - if (MO.getTargetFlags() & PPCII::MO_TPREL_FLAG) { + unsigned Flag = MO.getTargetFlags(); + if (Flag == PPCII::MO_TPREL_FLAG || + Flag == PPCII::MO_GOT_TPREL_PCREL_FLAG || + Flag == PPCII::MO_TPREL_PCREL_FLAG) { assert(MO.isGlobal() && "Only expecting a global MachineOperand here!\n"); TLSModel::Model Model = TM.getTLSModel(MO.getGlobal()); if (Model == TLSModel::LocalExec) @@ -842,9 +831,9 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { // For GD TLS access on AIX, we have two TOC entries for the symbol (one for // the variable offset and the other for the region handle). They are // differentiated by MO_TLSGD_FLAG and MO_TLSGDM_FLAG. - if (MO.getTargetFlags() & PPCII::MO_TLSGDM_FLAG) + if (Flag == PPCII::MO_TLSGDM_FLAG) return MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSGDM; - if (MO.getTargetFlags() & PPCII::MO_TLSGD_FLAG) + if (Flag == PPCII::MO_TLSGD_FLAG || Flag == PPCII::MO_GOT_TLSGD_PCREL_FLAG) return MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSGD; return MCSymbolRefExpr::VariantKind::VK_None; }; @@ -1538,8 +1527,10 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { // The faster non-TOC-based local-exec sequence is represented by `addi` // with an immediate operand having the MO_TPREL_FLAG. Such an instruction // does not otherwise arise. - const MachineOperand &MO = MI->getOperand(2); - if ((MO.getTargetFlags() & PPCII::MO_TPREL_FLAG) != 0) { + unsigned Flag = MI->getOperand(2).getTargetFlags(); + if (Flag == PPCII::MO_TPREL_FLAG || + Flag == PPCII::MO_GOT_TPREL_PCREL_FLAG || + Flag == PPCII::MO_TPREL_PCREL_FLAG) { assert( Subtarget->hasAIXSmallLocalExecTLS() && "addi with thread-pointer only expected with local-exec small TLS"); diff --git a/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def b/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def index 5d97d187b2969a38582ba1edeb186e4def73e2c8..8bbe315a2bb9a745cc46ae8c87bb4291ac73699c 100644 --- a/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def +++ b/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def @@ -958,6 +958,7 @@ FUSION_FEATURE(GeneralBack2Back, hasBack2BackFusion, -1, V_SET0B, V_SET0H, WAIT, + WAITP10, XOR, XOR8, XOR8_rec, diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index ca65048e2355f399baa97c4d49fd8af708405d1e..acaf98b62fd94d08be824a3cf8e0625401b1bb5f 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -2972,7 +2972,7 @@ bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base, template static bool isValidPCRelNode(SDValue N) { Ty *PCRelCand = dyn_cast(N); - return PCRelCand && (PCRelCand->getTargetFlags() & PPCII::MO_PCREL_FLAG); + return PCRelCand && (PPCInstrInfo::hasPCRelFlag(PCRelCand->getTargetFlags())); } /// Returns true if this address is a PC Relative address. @@ -3133,8 +3133,8 @@ static void getLabelAccessInfo(bool IsPIC, const PPCSubtarget &Subtarget, // Don't use the pic base if not in PIC relocation model. if (IsPIC) { - HiOpFlags |= PPCII::MO_PIC_FLAG; - LoOpFlags |= PPCII::MO_PIC_FLAG; + HiOpFlags = PPCII::MO_PIC_HA_FLAG; + LoOpFlags = PPCII::MO_PIC_LO_FLAG; } } @@ -3453,8 +3453,8 @@ SDValue PPCTargetLowering::LowerGlobalTLSAddressLinux(SDValue Op, if (Model == TLSModel::LocalExec) { if (Subtarget.isUsingPCRelativeCalls()) { SDValue TLSReg = DAG.getRegister(PPC::X13, MVT::i64); - SDValue TGA = DAG.getTargetGlobalAddress( - GV, dl, PtrVT, 0, (PPCII::MO_PCREL_FLAG | PPCII::MO_TPREL_FLAG)); + SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, + PPCII::MO_TPREL_PCREL_FLAG); SDValue MatAddr = DAG.getNode(PPCISD::TLS_LOCAL_EXEC_MAT_ADDR, dl, PtrVT, TGA); return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TLSReg, MatAddr); @@ -3476,8 +3476,7 @@ SDValue PPCTargetLowering::LowerGlobalTLSAddressLinux(SDValue Op, SDValue TGA = DAG.getTargetGlobalAddress( GV, dl, PtrVT, 0, IsPCRel ? PPCII::MO_GOT_TPREL_PCREL_FLAG : 0); SDValue TGATLS = DAG.getTargetGlobalAddress( - GV, dl, PtrVT, 0, - IsPCRel ? (PPCII::MO_TLS | PPCII::MO_PCREL_FLAG) : PPCII::MO_TLS); + GV, dl, PtrVT, 0, IsPCRel ? PPCII::MO_TLS_PCREL_FLAG : PPCII::MO_TLS); SDValue TPOffset; if (IsPCRel) { SDValue MatPCRel = DAG.getNode(PPCISD::MAT_PCREL_ADDR, dl, PtrVT, TGA); @@ -3573,8 +3572,7 @@ SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op, EVT Ty = getPointerTy(DAG.getDataLayout()); if (isAccessedAsGotIndirect(Op)) { SDValue GA = DAG.getTargetGlobalAddress(GV, DL, Ty, GSDN->getOffset(), - PPCII::MO_PCREL_FLAG | - PPCII::MO_GOT_FLAG); + PPCII::MO_GOT_PCREL_FLAG); SDValue MatPCRel = DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, GA); SDValue Load = DAG.getLoad(MVT::i64, DL, DAG.getEntryNode(), MatPCRel, MachinePointerInfo()); diff --git a/llvm/lib/Target/PowerPC/PPCInstrFormats.td b/llvm/lib/Target/PowerPC/PPCInstrFormats.td index 6501c17dd810e64613f9297a31cc2526dca56eb2..5389f42a325ce61e456cf805a2d0950564203ff7 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrFormats.td +++ b/llvm/lib/Target/PowerPC/PPCInstrFormats.td @@ -725,6 +725,38 @@ class XForm_24_sync opcode, bits<10> xo, dag OOL, dag IOL, let Inst{31} = 0; } +class XForm_IMM2_IMM2 opcode, bits<10> xo, dag OOL, dag IOL, + string asmstr, InstrItinClass itin, list pattern> + : I { + bits<2> L; + bits<2> PL; + + let Pattern = pattern; + let Inst{6-8} = 0; + let Inst{9-10} = L; + let Inst{11-13} = 0; + let Inst{14-15} = PL; + let Inst{16-20} = 0; + let Inst{21-30} = xo; + let Inst{31} = 0; +} + +class XForm_IMM3_IMM2 opcode, bits<10> xo, dag OOL, dag IOL, + string asmstr, InstrItinClass itin, list pattern> + : I { + bits<3> L; + bits<2> SC; + + let Pattern = pattern; + let Inst{6-7} = 0; + let Inst{8-10} = L; + let Inst{11-13} = 0; + let Inst{14-15} = SC; + let Inst{16-20} = 0; + let Inst{21-30} = xo; + let Inst{31} = 0; +} + class XForm_24_eieio opcode, bits<10> xo, dag OOL, dag IOL, string asmstr, InstrItinClass itin, list pattern> : XForm_24_sync { diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp index 21cd27d6271134064e5c0b228636e5485b3f8a33..d0a6cced1b1974c0d03192c35ac61d8eff47e800 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp @@ -2956,27 +2956,12 @@ unsigned PPCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { std::pair PPCInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { - const unsigned Mask = PPCII::MO_ACCESS_MASK; - return std::make_pair(TF & Mask, TF & ~Mask); + // PPC always uses a direct mask. + return std::make_pair(TF, 0u); } ArrayRef> PPCInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { - using namespace PPCII; - static const std::pair TargetFlags[] = { - {MO_LO, "ppc-lo"}, - {MO_HA, "ppc-ha"}, - {MO_TPREL_LO, "ppc-tprel-lo"}, - {MO_TPREL_HA, "ppc-tprel-ha"}, - {MO_DTPREL_LO, "ppc-dtprel-lo"}, - {MO_TLSLD_LO, "ppc-tlsld-lo"}, - {MO_TOC_LO, "ppc-toc-lo"}, - {MO_TLS, "ppc-tls"}}; - return ArrayRef(TargetFlags); -} - -ArrayRef> -PPCInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const { using namespace PPCII; static const std::pair TargetFlags[] = { {MO_PLT, "ppc-plt"}, @@ -2985,12 +2970,26 @@ PPCInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const { {MO_GOT_FLAG, "ppc-got"}, {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"}, {MO_TLSGD_FLAG, "ppc-tlsgd"}, - {MO_TLSLD_FLAG, "ppc-tlsld"}, {MO_TPREL_FLAG, "ppc-tprel"}, + {MO_TLSLD_FLAG, "ppc-tlsld"}, {MO_TLSGDM_FLAG, "ppc-tlsgdm"}, {MO_GOT_TLSGD_PCREL_FLAG, "ppc-got-tlsgd-pcrel"}, {MO_GOT_TLSLD_PCREL_FLAG, "ppc-got-tlsld-pcrel"}, - {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"}}; + {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"}, + {MO_LO, "ppc-lo"}, + {MO_HA, "ppc-ha"}, + {MO_TPREL_LO, "ppc-tprel-lo"}, + {MO_TPREL_HA, "ppc-tprel-ha"}, + {MO_DTPREL_LO, "ppc-dtprel-lo"}, + {MO_TLSLD_LO, "ppc-tlsld-lo"}, + {MO_TOC_LO, "ppc-toc-lo"}, + {MO_TLS, "ppc-tls"}, + {MO_PIC_HA_FLAG, "ppc-ha-pic"}, + {MO_PIC_LO_FLAG, "ppc-lo-pic"}, + {MO_TPREL_PCREL_FLAG, "ppc-tprel-pcrel"}, + {MO_TLS_PCREL_FLAG, "ppc-tls-pcrel"}, + {MO_GOT_PCREL_FLAG, "ppc-got-pcrel"}, + }; return ArrayRef(TargetFlags); } diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h index 2f0b9498411aee60bd755aecea0b68c1bfa087f2..75f9cd1c206d827d03566709747dafa9fb829674 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h @@ -14,6 +14,7 @@ #define LLVM_LIB_TARGET_POWERPC_PPCINSTRINFO_H #include "MCTargetDesc/PPCMCTargetDesc.h" +#include "PPC.h" #include "PPCRegisterInfo.h" #include "llvm/ADT/SmallSet.h" #include "llvm/CodeGen/TargetInstrInfo.h" @@ -283,6 +284,32 @@ public: return false; } + static bool hasPCRelFlag(unsigned TF) { + return TF == PPCII::MO_PCREL_FLAG || TF == PPCII::MO_GOT_TLSGD_PCREL_FLAG || + TF == PPCII::MO_GOT_TLSLD_PCREL_FLAG || + TF == PPCII::MO_GOT_TPREL_PCREL_FLAG || + TF == PPCII::MO_TPREL_PCREL_FLAG || TF == PPCII::MO_TLS_PCREL_FLAG || + TF == PPCII::MO_GOT_PCREL_FLAG; + } + + static bool hasGOTFlag(unsigned TF) { + return TF == PPCII::MO_GOT_FLAG || TF == PPCII::MO_GOT_TLSGD_PCREL_FLAG || + TF == PPCII::MO_GOT_TLSLD_PCREL_FLAG || + TF == PPCII::MO_GOT_TPREL_PCREL_FLAG || + TF == PPCII::MO_GOT_PCREL_FLAG; + } + + static bool hasTLSFlag(unsigned TF) { + return TF == PPCII::MO_TLSGD_FLAG || TF == PPCII::MO_TPREL_FLAG || + TF == PPCII::MO_TLSLD_FLAG || TF == PPCII::MO_TLSGDM_FLAG || + TF == PPCII::MO_GOT_TLSGD_PCREL_FLAG || + TF == PPCII::MO_GOT_TLSLD_PCREL_FLAG || + TF == PPCII::MO_GOT_TPREL_PCREL_FLAG || TF == PPCII::MO_TPREL_LO || + TF == PPCII::MO_TPREL_HA || TF == PPCII::MO_DTPREL_LO || + TF == PPCII::MO_TLSLD_LO || TF == PPCII::MO_TLS || + TF == PPCII::MO_TPREL_PCREL_FLAG || TF == PPCII::MO_TLS_PCREL_FLAG; + } + ScheduleHazardRecognizer * CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, const ScheduleDAG *DAG) const override; @@ -557,9 +584,6 @@ public: ArrayRef> getSerializableDirectMachineOperandTargetFlags() const override; - ArrayRef> - getSerializableBitmaskMachineOperandTargetFlags() const override; - // Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction. bool expandVSXMemPseudo(MachineInstr &MI) const; diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td index a97062e0c643fb2aa514ad9d88bfb7b3e217a021..6199785206b2f7601892edb233ae3f4a16eefa58 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -3437,6 +3437,23 @@ def crnot : OutPatFrag<(ops node:$in), def : Pat<(not i1:$in), (crnot $in)>; +// Pseudo-instructions for alternate assembly syntax (never used by codegen). +// These are aliases that require C++ handling to convert to the target +// instruction, while InstAliases can be handled directly by tblgen. +class PPCAsmPseudo + : Instruction { + let Namespace = "PPC"; + bit PPC64 = 0; // Default value, override with isPPC64 + + let OutOperandList = (outs); + let InOperandList = iops; + let Pattern = []; + let AsmString = asm; + let isAsmParserOnly = 1; + let isPseudo = 1; + let hasNoSchedulingInfo = 1; +} + // Prefixed instructions may require access to the above defs at a later // time so we include this after the def. include "PPCInstrP10.td" @@ -4458,23 +4475,6 @@ def ICBIEP : XForm_1a<31, 991, (outs), (ins (memrr $RA, $RB):$addr), "icbiep $a // PowerPC Assembler Instruction Aliases // -// Pseudo-instructions for alternate assembly syntax (never used by codegen). -// These are aliases that require C++ handling to convert to the target -// instruction, while InstAliases can be handled directly by tblgen. -class PPCAsmPseudo - : Instruction { - let Namespace = "PPC"; - bit PPC64 = 0; // Default value, override with isPPC64 - - let OutOperandList = (outs); - let InOperandList = iops; - let Pattern = []; - let AsmString = asm; - let isAsmParserOnly = 1; - let isPseudo = 1; - let hasNoSchedulingInfo = 1; -} - def : InstAlias<"sc", (SC 0)>; def : InstAlias<"sync", (SYNC 0)>, Requires<[HasSYNC]>; diff --git a/llvm/lib/Target/PowerPC/PPCInstrP10.td b/llvm/lib/Target/PowerPC/PPCInstrP10.td index f72a52d5b26f6b3d9ab8dd8ee16f09d14931004c..d5a372e4dc101086352c58695485ba4b332090f2 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrP10.td +++ b/llvm/lib/Target/PowerPC/PPCInstrP10.td @@ -575,33 +575,54 @@ class XForm_XT5_BI5 opcode, bits<10> xo, dag OOL, dag IOL, } multiclass MLS_DForm_R_SI34_RTA5_MEM_p opcode, dag OOL, dag IOL, - dag PCRel_IOL, string asmstr, + dag PCRel_IOL, dag PCRelOnly_IOL, + string asmstr, string asmstr_pcext, InstrItinClass itin> { def NAME : MLS_DForm_R_SI34_RTA5_MEM; def pc : MLS_DForm_R_SI34_RTA5_MEM, isPCRel; + let isAsmParserOnly = 1, hasNoSchedulingInfo = 1 in { + def nopc : MLS_DForm_R_SI34_RTA5_MEM; + let RA = 0 in + def onlypc : MLS_DForm_R_SI34_RTA5_MEM, isPCRel; + } } multiclass 8LS_DForm_R_SI34_RTA5_MEM_p opcode, dag OOL, dag IOL, - dag PCRel_IOL, string asmstr, + dag PCRel_IOL, dag PCRelOnly_IOL, + string asmstr, string asmstr_pcext, InstrItinClass itin> { def NAME : 8LS_DForm_R_SI34_RTA5_MEM; def pc : 8LS_DForm_R_SI34_RTA5_MEM, isPCRel; + let isAsmParserOnly = 1, hasNoSchedulingInfo = 1 in { + def nopc : 8LS_DForm_R_SI34_RTA5_MEM; + let RA = 0 in + def onlypc : 8LS_DForm_R_SI34_RTA5_MEM, isPCRel; + } } multiclass 8LS_DForm_R_SI34_XT6_RA5_MEM_p opcode, dag OOL, dag IOL, - dag PCRel_IOL, string asmstr, + dag PCRel_IOL, dag PCRelOnly_IOL, + string asmstr, string asmstr_pcext, InstrItinClass itin> { def NAME : 8LS_DForm_R_SI34_XT6_RA5_MEM; def pc : 8LS_DForm_R_SI34_XT6_RA5_MEM, isPCRel; + let isAsmParserOnly = 1, hasNoSchedulingInfo = 1 in { + def nopc : 8LS_DForm_R_SI34_XT6_RA5_MEM; + let RA = 0 in + def onlypc : 8LS_DForm_R_SI34_XT6_RA5_MEM, isPCRel; + } } def PrefixInstrs : Predicate<"Subtarget->hasPrefixInstrs()">; @@ -638,68 +659,88 @@ let Predicates = [PrefixInstrs] in { defm PLXV : 8LS_DForm_R_SI34_XT6_RA5_MEM_p<25, (outs vsrc:$XST), (ins (memri34 $D, $RA):$addr), (ins (memri34_pcrel $D, $RA):$addr), - "plxv $XST, $addr", IIC_LdStLFD>; + (ins s34imm_pcrel:$D), + "plxv $XST, $addr", "plxv $XST, $D", IIC_LdStLFD>; defm PLFS : MLS_DForm_R_SI34_RTA5_MEM_p<48, (outs f4rc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plfs $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plfs $RST, $addr", + "plfs $RST, $D", IIC_LdStLFD>; defm PLFD : MLS_DForm_R_SI34_RTA5_MEM_p<50, (outs f8rc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plfd $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plfd $RST, $addr", + "plfd $RST, $D", IIC_LdStLFD>; defm PLXSSP : 8LS_DForm_R_SI34_RTA5_MEM_p<43, (outs vfrc:$RST), (ins (memri34 $D, $RA):$addr), (ins (memri34_pcrel $D, $RA):$addr), - "plxssp $RST, $addr", IIC_LdStLFD>; + (ins s34imm_pcrel:$D), + "plxssp $RST, $addr", "plxssp $RST, $D", + IIC_LdStLFD>; defm PLXSD : 8LS_DForm_R_SI34_RTA5_MEM_p<42, (outs vfrc:$RST), (ins (memri34 $D, $RA):$addr), (ins (memri34_pcrel $D, $RA):$addr), - "plxsd $RST, $addr", IIC_LdStLFD>; + (ins s34imm_pcrel:$D), + "plxsd $RST, $addr", "plxsd $RST, $D", + IIC_LdStLFD>; let Interpretation64Bit = 1, isCodeGenOnly = 1 in { defm PLBZ8 : MLS_DForm_R_SI34_RTA5_MEM_p<34, (outs g8rc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plbz $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plbz $RST, $addr", + "plbz $RST, $D", IIC_LdStLFD>; defm PLHZ8 : MLS_DForm_R_SI34_RTA5_MEM_p<40, (outs g8rc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plhz $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plhz $RST, $addr", + "plhz $RST, $D", IIC_LdStLFD>; defm PLHA8 : MLS_DForm_R_SI34_RTA5_MEM_p<42, (outs g8rc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plha $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plha $RST, $addr", + "plha $RST, $D", IIC_LdStLFD>; defm PLWA8 : 8LS_DForm_R_SI34_RTA5_MEM_p<41, (outs g8rc:$RST), (ins (memri34 $D, $RA):$addr), (ins (memri34_pcrel $D, $RA):$addr), - "plwa $RST, $addr", IIC_LdStLFD>; + (ins s34imm_pcrel:$D), + "plwa $RST, $addr", "plwa $RST, $D", IIC_LdStLFD>; defm PLWZ8 : MLS_DForm_R_SI34_RTA5_MEM_p<32, (outs g8rc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plwz $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plwz $RST, $addr", + "plwz $RST, $D", IIC_LdStLFD>; } defm PLBZ : MLS_DForm_R_SI34_RTA5_MEM_p<34, (outs gprc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plbz $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plbz $RST, $addr", + "plbz $RST, $D", IIC_LdStLFD>; defm PLHZ : MLS_DForm_R_SI34_RTA5_MEM_p<40, (outs gprc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plhz $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plhz $RST, $addr", + "plhz $RST, $D", IIC_LdStLFD>; defm PLHA : MLS_DForm_R_SI34_RTA5_MEM_p<42, (outs gprc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plha $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plha $RST, $addr", + "plha $RST, $D", IIC_LdStLFD>; defm PLWZ : MLS_DForm_R_SI34_RTA5_MEM_p<32, (outs gprc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plwz $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plwz $RST, $addr", + "plwz $RST, $D", IIC_LdStLFD>; defm PLWA : 8LS_DForm_R_SI34_RTA5_MEM_p<41, (outs gprc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plwa $RST, $addr", + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), + "plwa $RST, $addr", "plwa $RST, $D", IIC_LdStLFD>; defm PLD : 8LS_DForm_R_SI34_RTA5_MEM_p<57, (outs g8rc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "pld $RST, $addr", + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), + "pld $RST, $addr", "pld $RST, $D", IIC_LdStLFD>; } @@ -707,53 +748,65 @@ let Predicates = [PrefixInstrs] in { defm PSTXV : 8LS_DForm_R_SI34_XT6_RA5_MEM_p<27, (outs), (ins vsrc:$XST, (memri34 $D, $RA):$addr), (ins vsrc:$XST, (memri34_pcrel $D, $RA):$addr), - "pstxv $XST, $addr", IIC_LdStLFD>; + (ins vsrc:$XST, s34imm_pcrel:$D), + "pstxv $XST, $addr", "pstxv $XST, $D", IIC_LdStLFD>; defm PSTFS : MLS_DForm_R_SI34_RTA5_MEM_p<52, (outs), (ins f4rc:$RST, (memri34 $D, $RA):$addr), (ins f4rc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstfs $RST, $addr", IIC_LdStLFD>; + (ins f4rc:$RST, s34imm_pcrel:$D), + "pstfs $RST, $addr", "pstfs $RST, $D", IIC_LdStLFD>; defm PSTFD : MLS_DForm_R_SI34_RTA5_MEM_p<54, (outs), (ins f8rc:$RST, (memri34 $D, $RA):$addr), (ins f8rc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstfd $RST, $addr", IIC_LdStLFD>; + (ins f8rc:$RST, s34imm_pcrel:$D), + "pstfd $RST, $addr", "pstfd $RST, $D", IIC_LdStLFD>; defm PSTXSSP : 8LS_DForm_R_SI34_RTA5_MEM_p<47, (outs), (ins vfrc:$RST, (memri34 $D, $RA):$addr), (ins vfrc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstxssp $RST, $addr", IIC_LdStLFD>; + (ins vfrc:$RST, s34imm_pcrel:$D), + "pstxssp $RST, $addr", "pstxssp $RST, $D", IIC_LdStLFD>; defm PSTXSD : 8LS_DForm_R_SI34_RTA5_MEM_p<46, (outs), (ins vfrc:$RST, (memri34 $D, $RA):$addr), (ins vfrc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstxsd $RST, $addr", IIC_LdStLFD>; + (ins vfrc:$RST, s34imm_pcrel:$D), + "pstxsd $RST, $addr", "pstxsd $RST, $D", IIC_LdStLFD>; let Interpretation64Bit = 1, isCodeGenOnly = 1 in { defm PSTB8 : MLS_DForm_R_SI34_RTA5_MEM_p<38, (outs), (ins g8rc:$RST, (memri34 $D, $RA):$addr), (ins g8rc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstb $RST, $addr", IIC_LdStLFD>; + (ins g8rc:$RST, s34imm_pcrel:$D), + "pstb $RST, $addr", "pstb $RST, $D", IIC_LdStLFD>; defm PSTH8 : MLS_DForm_R_SI34_RTA5_MEM_p<44, (outs), (ins g8rc:$RST, (memri34 $D, $RA):$addr), (ins g8rc:$RST, (memri34_pcrel $D, $RA):$addr), - "psth $RST, $addr", IIC_LdStLFD>; + (ins g8rc:$RST, s34imm_pcrel:$D), + "psth $RST, $addr", "psth $RST, $D", IIC_LdStLFD>; defm PSTW8 : MLS_DForm_R_SI34_RTA5_MEM_p<36, (outs), (ins g8rc:$RST, (memri34 $D, $RA):$addr), (ins g8rc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstw $RST, $addr", IIC_LdStLFD>; + (ins g8rc:$RST, s34imm_pcrel:$D), + "pstw $RST, $addr", "pstw $RST, $D", IIC_LdStLFD>; } defm PSTB : MLS_DForm_R_SI34_RTA5_MEM_p<38, (outs), (ins gprc:$RST, (memri34 $D, $RA):$addr), (ins gprc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstb $RST, $addr", IIC_LdStLFD>; + (ins gprc:$RST, s34imm_pcrel:$D), + "pstb $RST, $addr", "pstb $RST, $D", IIC_LdStLFD>; defm PSTH : MLS_DForm_R_SI34_RTA5_MEM_p<44, (outs), (ins gprc:$RST, (memri34 $D, $RA):$addr), (ins gprc:$RST, (memri34_pcrel $D, $RA):$addr), - "psth $RST, $addr", IIC_LdStLFD>; + (ins gprc:$RST, s34imm_pcrel:$D), + "psth $RST, $addr", "psth $RST, $D", IIC_LdStLFD>; defm PSTW : MLS_DForm_R_SI34_RTA5_MEM_p<36, (outs), (ins gprc:$RST, (memri34 $D, $RA):$addr), (ins gprc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstw $RST, $addr", IIC_LdStLFD>; + (ins gprc:$RST, s34imm_pcrel:$D), + "pstw $RST, $addr", "pstw $RST, $D", IIC_LdStLFD>; defm PSTD : 8LS_DForm_R_SI34_RTA5_MEM_p<61, (outs), (ins g8rc:$RST, (memri34 $D, $RA):$addr), (ins g8rc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstd $RST, $addr", IIC_LdStLFD>; + (ins g8rc:$RST, s34imm_pcrel:$D), + "pstd $RST, $addr", "pstd $RST, $D", IIC_LdStLFD>; } } @@ -812,13 +865,20 @@ class 8LS_DForm_R_XTp5_SI34_MEM opcode, dag OOL, dag IOL, string asmstr, } multiclass 8LS_DForm_R_XTp5_SI34_MEM_p opcode, dag OOL, - dag IOL, dag PCRel_IOL, - string asmstr, InstrItinClass itin> { + dag IOL, dag PCRel_IOL, dag PCRelOnly_IOL, + string asmstr, string asmstr_pcext, + InstrItinClass itin> { def NAME : 8LS_DForm_R_XTp5_SI34_MEM; def pc : 8LS_DForm_R_XTp5_SI34_MEM, isPCRel; + let isAsmParserOnly = 1, hasNoSchedulingInfo = 1 in { + def nopc : 8LS_DForm_R_XTp5_SI34_MEM; + let RA = 0 in + def onlypc : 8LS_DForm_R_XTp5_SI34_MEM, isPCRel; + } } @@ -1079,7 +1139,9 @@ let mayLoad = 0, mayStore = 1, Predicates = [PairedVectorMemops] in { let mayLoad = 1, mayStore = 0, Predicates = [PairedVectorMemops, PrefixInstrs] in { defm PLXVP : 8LS_DForm_R_XTp5_SI34_MEM_p<58, (outs vsrprc:$XTp), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plxvp $XTp, $addr", + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), + "plxvp $XTp, $addr", "plxvp $XTp, $D", IIC_LdStLFD>; } @@ -1087,7 +1149,8 @@ let mayLoad = 0, mayStore = 1, Predicates = [PairedVectorMemops, PrefixInstrs] i defm PSTXVP : 8LS_DForm_R_XTp5_SI34_MEM_p<62, (outs), (ins vsrprc:$XTp, (memri34 $D, $RA):$addr), (ins vsrprc:$XTp, (memri34_pcrel $D, $RA):$addr), - "pstxvp $XTp, $addr", IIC_LdStLFD>; + (ins vsrprc:$XTp, s34imm_pcrel:$D), + "pstxvp $XTp, $addr", "pstxvp $XTp, $D", IIC_LdStLFD>; } let Predicates = [PairedVectorMemops] in { @@ -1855,6 +1918,13 @@ let Predicates = [IsISA3_1, HasVSX] in { [(set f128:$RST, (PPCxsminc f128:$RA, f128:$RB))]>; } +let Predicates = [IsISA3_1] in { + def WAITP10 : XForm_IMM2_IMM2<31, 30, (outs), (ins u2imm:$L, u2imm:$PL), + "wait $L $PL", IIC_LdStLoad, []>; + def SYNCP10 : XForm_IMM3_IMM2<31, 598, (outs), (ins u3imm:$L, u2imm:$SC), + "sync $L, $SC", IIC_LdStSync, []>; +} + // Multiclass defining patterns for Set Boolean Extension Reverse Instructions. // This is analogous to the CRNotPat multiclass but specifically for Power10 // and newer subtargets since the extended forms use Set Boolean instructions. @@ -2386,3 +2456,50 @@ let AddedComplexity = 400, Predicates = [IsISA3_1, HasVSX, IsBigEndian] in { def : Pat<(v2i64 (insertelt v2i64:$vDi, i64:$rA, Idx)), (VINSD $vDi, !mul(Idx, 8), $rA)>; } + + +//===----------------------------------------------------------------------===// +// PowerPC ISA 3.1 Extended Mnemonics. +// + +let Predicates = [IsISA3_1] in { + def : InstAlias<"wait", (WAITP10 0, 0)>; + def : InstAlias<"wait 0", (WAITP10 0, 0), 0>; + def : InstAlias<"wait 1", (WAITP10 1, 0), 0>; + def : InstAlias<"waitrsv", (WAITP10 1, 0)>; + def : InstAlias<"pause_short", (WAITP10 2, 0), 0>; + + def : InstAlias<"sync", (SYNCP10 0, 0)>; + def : InstAlias<"hwsync", (SYNCP10 0, 0), 0>; + def : InstAlias<"wsync", (SYNCP10 1, 0), 0>; + def : InstAlias<"ptesync", (SYNCP10 2, 0)>; + def : InstAlias<"phwsync", (SYNCP10 4, 0)>; + def : InstAlias<"plwsync", (SYNCP10 5, 0)>; + def : InstAlias<"sync $L", (SYNCP10 u3imm:$L, 0)>; + def : InstAlias<"stncisync", (SYNCP10 1, 1)>; + def : InstAlias<"stcisync", (SYNCP10 0, 2)>; + def : InstAlias<"stsync", (SYNCP10 0, 3)>; + + def : InstAlias<"paddi $RT, $RA, $SI", (PADDI8 g8rc:$RT, g8rc_nox0:$RA, s34imm:$SI)>; +} + +let Predicates = [IsISA3_1, PrefixInstrs], isAsmParserOnly = 1, hasNoSchedulingInfo = 1 in { + let Interpretation64Bit = 1 in { + def PLA8 : MLS_DForm_SI34_RT5<14, (outs g8rc:$RT), + (ins g8rc_nox0:$RA, s34imm:$SI), + "pla $RT, ${SI} ${RA}", IIC_IntSimple, []>; + def PLA8pc : MLS_DForm_SI34_RT5<14, (outs g8rc:$RT), + (ins s34imm_pcrel:$SI), + "pla $RT, $SI", IIC_IntSimple, []>, isPCRel; + } + + def PSUBI : PPCAsmPseudo<"psubi $RT, $RA, $SI", + (ins g8rc:$RT, g8rc_nox0:$RA, s34imm:$SI)>; + + def PLA : MLS_DForm_SI34_RT5<14, (outs gprc:$RT), + (ins gprc_nor0:$RA, s34imm:$SI), + "pla $RT, ${SI} ${RA}", IIC_IntSimple, []>; + def PLApc : MLS_DForm_SI34_RT5<14, (outs gprc:$RT), + (ins s34imm_pcrel:$SI), + "pla $RT, $SI", IIC_IntSimple, []>, isPCRel; +} diff --git a/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp b/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp index c9797fdc22863df4924676c965ac90e6ab83c7aa..9a3ca5a7829362bf3d1eac53d90f0d247d935fdb 100644 --- a/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp +++ b/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp @@ -54,7 +54,7 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol, MCContext &Ctx = Printer.OutContext; MCSymbolRefExpr::VariantKind RefKind = MCSymbolRefExpr::VK_None; - unsigned access = MO.getTargetFlags() & PPCII::MO_ACCESS_MASK; + unsigned access = MO.getTargetFlags(); switch (access) { case PPCII::MO_TPREL_LO: @@ -73,9 +73,10 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol, RefKind = MCSymbolRefExpr::VK_PPC_TOC_LO; break; case PPCII::MO_TLS: - bool IsPCRel = (MO.getTargetFlags() & ~access) == PPCII::MO_PCREL_FLAG; - RefKind = IsPCRel ? MCSymbolRefExpr::VK_PPC_TLS_PCREL - : MCSymbolRefExpr::VK_PPC_TLS; + RefKind = MCSymbolRefExpr::VK_PPC_TLS; + break; + case PPCII::MO_TLS_PCREL_FLAG: + RefKind = MCSymbolRefExpr::VK_PPC_TLS_PCREL; break; } @@ -85,9 +86,9 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol, RefKind = MCSymbolRefExpr::VK_PLT; else if (MO.getTargetFlags() == PPCII::MO_PCREL_FLAG) RefKind = MCSymbolRefExpr::VK_PCREL; - else if (MO.getTargetFlags() == (PPCII::MO_PCREL_FLAG | PPCII::MO_GOT_FLAG)) + else if (MO.getTargetFlags() == PPCII::MO_GOT_PCREL_FLAG) RefKind = MCSymbolRefExpr::VK_PPC_GOT_PCREL; - else if (MO.getTargetFlags() == (PPCII::MO_PCREL_FLAG | PPCII::MO_TPREL_FLAG)) + else if (MO.getTargetFlags() == PPCII::MO_TPREL_PCREL_FLAG) RefKind = MCSymbolRefExpr::VK_TPREL; else if (MO.getTargetFlags() == PPCII::MO_GOT_TLSGD_PCREL_FLAG) RefKind = MCSymbolRefExpr::VK_PPC_GOT_TLSGD_PCREL; @@ -138,7 +139,9 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol, Ctx); // Subtract off the PIC base if required. - if (MO.getTargetFlags() & PPCII::MO_PIC_FLAG) { + if (MO.getTargetFlags() == PPCII::MO_PIC_FLAG || + MO.getTargetFlags() == PPCII::MO_PIC_HA_FLAG || + MO.getTargetFlags() == PPCII::MO_PIC_LO_FLAG) { const MachineFunction *MF = MO.getParent()->getParent()->getParent(); const MCExpr *PB = MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx); @@ -148,9 +151,11 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol, // Add ha16() / lo16() markers if required. switch (access) { case PPCII::MO_LO: + case PPCII::MO_PIC_LO_FLAG: Expr = PPCMCExpr::createLo(Expr, Ctx); break; case PPCII::MO_HA: + case PPCII::MO_PIC_HA_FLAG: Expr = PPCMCExpr::createHa(Expr, Ctx); break; } diff --git a/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp b/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp index 6ad69fffea87a2e164ffb7e2b0b3a479c67eab87..6e3bf26a598a9a6d121f828abe4e95602d1c7ad3 100644 --- a/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp +++ b/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp @@ -239,7 +239,7 @@ static bool hasPCRelativeForm(MachineInstr &Use) { return false; // Finally return true only if the GOT flag is present. - return (SymbolOp.getTargetFlags() & PPCII::MO_GOT_FLAG); + return PPCInstrInfo::hasGOTFlag(SymbolOp.getTargetFlags()); } bool addLinkerOpt(MachineBasicBlock &MBB, const TargetRegisterInfo *TRI) { diff --git a/llvm/lib/Target/PowerPC/PPCTOCRegDeps.cpp b/llvm/lib/Target/PowerPC/PPCTOCRegDeps.cpp index 8120975c4fb257a43bb96f3f0a6bde2c44e10c88..81f078ab246e6ab0bf2d2c0cb12f9ee18faf8afd 100644 --- a/llvm/lib/Target/PowerPC/PPCTOCRegDeps.cpp +++ b/llvm/lib/Target/PowerPC/PPCTOCRegDeps.cpp @@ -100,7 +100,7 @@ protected: return true; for (const MachineOperand &MO : MI.operands()) { - if ((MO.getTargetFlags() & PPCII::MO_ACCESS_MASK) == PPCII::MO_TOC_LO) + if (MO.getTargetFlags() == PPCII::MO_TOC_LO) return true; } diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp index aa385d7c3b202ad8e5018bb81303f0fa5f15840f..062b53e24a0d793055c1a3acc5cbf609bec41b4d 100644 --- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp @@ -18,7 +18,6 @@ #include "llvm/IR/ProfDataUtils.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/KnownBits.h" #include "llvm/Transforms/InstCombine/InstCombiner.h" #include "llvm/Transforms/Utils/Local.h" #include diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp index 570b9802a2f512a8f1549f626e433cc342ed0112..8f03a7ac41d37b04a6e78a4e378ea117a47a3e91 100644 --- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp +++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp @@ -13,6 +13,7 @@ #include "RISCVLegalizerInfo.h" #include "RISCVMachineFunctionInfo.h" #include "RISCVSubtarget.h" +#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" @@ -317,9 +318,60 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) getActionDefinitionsBuilder(G_VASTART).customFor({p0}); + // va_list must be a pointer, but most sized types are pretty easy to handle + // as the destination. + getActionDefinitionsBuilder(G_VAARG) + // TODO: Implement narrowScalar and widenScalar for G_VAARG for types + // outside the [s32, sXLen] range. + .clampScalar(0, s32, sXLen) + .lowerForCartesianProduct({s32, sXLen, p0}, {p0}); + getLegacyLegalizerInfo().computeTables(); } +static Type *getTypeForLLT(LLT Ty, LLVMContext &C) { + if (Ty.isVector()) + return FixedVectorType::get(IntegerType::get(C, Ty.getScalarSizeInBits()), + Ty.getNumElements()); + return IntegerType::get(C, Ty.getSizeInBits()); +} + +bool RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, + MachineInstr &MI) const { + Intrinsic::ID IntrinsicID = cast(MI).getIntrinsicID(); + switch (IntrinsicID) { + default: + return false; + case Intrinsic::vacopy: { + // vacopy arguments must be legal because of the intrinsic signature. + // No need to check here. + + MachineIRBuilder &MIRBuilder = Helper.MIRBuilder; + MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); + MachineFunction &MF = *MI.getMF(); + const DataLayout &DL = MIRBuilder.getDataLayout(); + LLVMContext &Ctx = MF.getFunction().getContext(); + + Register DstLst = MI.getOperand(1).getReg(); + LLT PtrTy = MRI.getType(DstLst); + + // Load the source va_list + Align Alignment = DL.getABITypeAlign(getTypeForLLT(PtrTy, Ctx)); + MachineMemOperand *LoadMMO = MF.getMachineMemOperand( + MachinePointerInfo(), MachineMemOperand::MOLoad, PtrTy, Alignment); + auto Tmp = MIRBuilder.buildLoad(PtrTy, MI.getOperand(2), *LoadMMO); + + // Store the result in the destination va_list + MachineMemOperand *StoreMMO = MF.getMachineMemOperand( + MachinePointerInfo(), MachineMemOperand::MOStore, PtrTy, Alignment); + MIRBuilder.buildStore(DstLst, Tmp, *StoreMMO); + + MI.eraseFromParent(); + return true; + } + } +} + bool RISCVLegalizerInfo::legalizeShlAshrLshr( MachineInstr &MI, MachineIRBuilder &MIRBuilder, GISelChangeObserver &Observer) const { @@ -354,7 +406,7 @@ bool RISCVLegalizerInfo::legalizeVAStart(MachineInstr &MI, LLT AddrTy = MIRBuilder.getMRI()->getType(MI.getOperand(0).getReg()); auto FINAddr = MIRBuilder.buildFrameIndex(AddrTy, FI); assert(MI.hasOneMemOperand()); - MIRBuilder.buildStore(MI.getOperand(0).getReg(), FINAddr, + MIRBuilder.buildStore(FINAddr, MI.getOperand(0).getReg(), *MI.memoperands()[0]); MI.eraseFromParent(); return true; diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h index 246ea90dcd74903c7caf88e8db815cfed17a90bc..48c36976501fc5f9b8265bb28a47de1e1443956d 100644 --- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h +++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h @@ -32,6 +32,9 @@ public: bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI) const override; + bool legalizeIntrinsic(LegalizerHelper &Helper, + MachineInstr &MI) const override; + private: bool legalizeShlAshrLshr(MachineInstr &MI, MachineIRBuilder &MIRBuilder, GISelChangeObserver &Observer) const; diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp index dfc3c9e9908d888bdaabee621d0797aa97ad3d43..716fb67c582489a5612f9ae7b69100e949228e78 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp @@ -108,7 +108,8 @@ RISCVAsmBackend::getFixupKindInfo(MCFixupKind Kind) const { // necessary for correctness as offsets may change during relaxation. bool RISCVAsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo *STI) { if (Fixup.getKind() >= FirstLiteralRelocationKind) return true; switch (Fixup.getTargetKind()) { @@ -128,7 +129,7 @@ bool RISCVAsmBackend::shouldForceRelocation(const MCAssembler &Asm, return true; } - return STI.hasFeature(RISCV::FeatureRelax) || ForceRelocs; + return STI->hasFeature(RISCV::FeatureRelax) || ForceRelocs; } bool RISCVAsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup, @@ -514,8 +515,8 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value, bool RISCVAsmBackend::evaluateTargetFixup( const MCAssembler &Asm, const MCAsmLayout &Layout, const MCFixup &Fixup, - const MCFragment *DF, const MCValue &Target, uint64_t &Value, - bool &WasForced) { + const MCFragment *DF, const MCValue &Target, const MCSubtargetInfo *STI, + uint64_t &Value, bool &WasForced) { const MCFixup *AUIPCFixup; const MCFragment *AUIPCDF; MCValue AUIPCTarget; @@ -565,7 +566,7 @@ bool RISCVAsmBackend::evaluateTargetFixup( Value = Layout.getSymbolOffset(SA) + AUIPCTarget.getConstant(); Value -= Layout.getFragmentOffset(AUIPCDF) + AUIPCFixup->getOffset(); - if (shouldForceRelocation(Asm, *AUIPCFixup, AUIPCTarget)) { + if (shouldForceRelocation(Asm, *AUIPCFixup, AUIPCTarget, STI)) { WasForced = true; return false; } diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h index 99b0d7b223b9937a97b5ee70a83e5381a8548dce..2ad6534ac8bce34c8cc010dad113384e0b56606a 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h @@ -50,8 +50,8 @@ public: bool evaluateTargetFixup(const MCAssembler &Asm, const MCAsmLayout &Layout, const MCFixup &Fixup, const MCFragment *DF, - const MCValue &Target, uint64_t &Value, - bool &WasForced) override; + const MCValue &Target, const MCSubtargetInfo *STI, + uint64_t &Value, bool &WasForced) override; bool handleAddSubRelocations(const MCAsmLayout &Layout, const MCFragment &F, const MCFixup &Fixup, const MCValue &Target, @@ -66,7 +66,8 @@ public: createObjectTargetWriter() const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value, const MCRelaxableFragment *DF, diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h index ad1713cad64cfec86fcb991372bbf3b3e1b22dfc..9eb18099894b2177e67282f95c180bb9b3d0915b 100644 --- a/llvm/lib/Target/RISCV/RISCV.h +++ b/llvm/lib/Target/RISCV/RISCV.h @@ -18,13 +18,8 @@ #include "llvm/Target/TargetMachine.h" namespace llvm { -class AsmPrinter; class FunctionPass; class InstructionSelector; -class MCInst; -class MCOperand; -class MachineInstr; -class MachineOperand; class PassRegistry; class RISCVRegisterBankInfo; class RISCVSubtarget; diff --git a/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp b/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp index 7099f36dcd43c22e2df1f335ac6395e9214ae3a8..df607236f7d56fffa0db1746d31ac2f0433dfec0 100644 --- a/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp +++ b/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp @@ -28,10 +28,7 @@ class RISCVDeadRegisterDefinitions : public MachineFunctionPass { public: static char ID; - RISCVDeadRegisterDefinitions() : MachineFunctionPass(ID) { - initializeRISCVDeadRegisterDefinitionsPass( - *PassRegistry::getPassRegistry()); - } + RISCVDeadRegisterDefinitions() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp index aa1bdbd8ba9c4345fa1df929331c110a3f6898e1..24a13f93af880ea84f8e6ffbbf4336e9ca5388b1 100644 --- a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp +++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp @@ -34,9 +34,7 @@ public: const RISCVInstrInfo *TII; static char ID; - RISCVExpandPseudo() : MachineFunctionPass(ID) { - initializeRISCVExpandPseudoPass(*PassRegistry::getPassRegistry()); - } + RISCVExpandPseudo() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; @@ -381,9 +379,7 @@ public: const RISCVInstrInfo *TII; static char ID; - RISCVPreRAExpandPseudo() : MachineFunctionPass(ID) { - initializeRISCVPreRAExpandPseudoPass(*PassRegistry::getPassRegistry()); - } + RISCVPreRAExpandPseudo() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index 7d142d38d0f9adeb8fa87b70ec666da4c4d7cdc6..294927aecb94b84ed57c40c0dc46058a7cf9d2a4 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -970,6 +970,16 @@ def TuneLUIADDIFusion : SubtargetFeature<"lui-addi-fusion", "HasLUIADDIFusion", "true", "Enable LUI+ADDI macrofusion">; +def TuneAUIPCADDIFusion + : SubtargetFeature<"auipc-addi-fusion", "HasAUIPCADDIFusion", + "true", "Enable AUIPC+ADDI macrofusion">; +def TuneShiftedZExtFusion + : SubtargetFeature<"shifted-zext-fusion", "HasShiftedZExtFusion", + "true", "Enable SLLI+SRLI to be fused when computing (shifted) zero extension">; +def TuneLDADDFusion + : SubtargetFeature<"ld-add-fusion", "HasLDADDFusion", + "true", "Enable LD+ADD macrofusion.">; + def TuneNoDefaultUnroll : SubtargetFeature<"no-default-unroll", "EnableDefaultUnroll", "false", "Disable default unroll preference.">; @@ -987,9 +997,12 @@ def TuneSiFive7 : SubtargetFeature<"sifive7", "RISCVProcFamily", "SiFive7", [TuneNoDefaultUnroll, TuneShortForwardBranchOpt]>; -def TuneVentanaVeyron : SubtargetFeature<"ventana-veyron", "RISCVProcFamily", "VentanaVeyron", - "Ventana-Veyron Series processors", - [TuneLUIADDIFusion]>; +def TuneVeyronFusions : SubtargetFeature<"ventana-veyron", "RISCVProcFamily", "VentanaVeyron", + "Ventana Veyron-Series processors", + [TuneLUIADDIFusion, + TuneAUIPCADDIFusion, + TuneShiftedZExtFusion, + TuneLDADDFusion]>; // Assume that lock-free native-width atomics are available, even if the target // and operating system combination would not usually provide them. The user diff --git a/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp b/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp index 871fc76d7d988a67985e7b95326d1db481482e07..6ee006525df560f20cb0ed031f0b209ab8709330 100644 --- a/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp +++ b/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp @@ -36,9 +36,7 @@ public: const TargetInstrInfo *TII; MachineRegisterInfo *MRI; const TargetRegisterInfo *TRI; - RISCVFoldMasks() : MachineFunctionPass(ID) { - initializeRISCVFoldMasksPass(*PassRegistry::getPassRegistry()); - } + RISCVFoldMasks() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; MachineFunctionProperties getRequiredProperties() const override { diff --git a/llvm/lib/Target/RISCV/RISCVInsertReadWriteCSR.cpp b/llvm/lib/Target/RISCV/RISCVInsertReadWriteCSR.cpp index acd19bf1b8a162ea0ded6c4bca54b642e681f8d2..b807abcc56819bd2ea520bbbfeac12f627503fac 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertReadWriteCSR.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertReadWriteCSR.cpp @@ -31,9 +31,7 @@ class RISCVInsertReadWriteCSR : public MachineFunctionPass { public: static char ID; - RISCVInsertReadWriteCSR() : MachineFunctionPass(ID) { - initializeRISCVInsertReadWriteCSRPass(*PassRegistry::getPassRegistry()); - } + RISCVInsertReadWriteCSR() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp index 323a92cfb8c83d3910bc56c20b6d20fb59cc3b01..ed3489f6db1c4ba6684dae62ee46a88180724833 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp @@ -742,9 +742,7 @@ class RISCVInsertVSETVLI : public MachineFunctionPass { public: static char ID; - RISCVInsertVSETVLI() : MachineFunctionPass(ID) { - initializeRISCVInsertVSETVLIPass(*PassRegistry::getPassRegistry()); - } + RISCVInsertVSETVLI() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; void getAnalysisUsage(AnalysisUsage &AU) const override { diff --git a/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp b/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp index 70c2415465f8cc4024725369ae1ba89bb6cdbabe..de2227f8219238b748b592407da650bdc15ea6fc 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp @@ -171,9 +171,7 @@ class RISCVInsertWriteVXRM : public MachineFunctionPass { public: static char ID; - RISCVInsertWriteVXRM() : MachineFunctionPass(ID) { - initializeRISCVInsertWriteVXRMPass(*PassRegistry::getPassRegistry()); - } + RISCVInsertWriteVXRM() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index 127d3080491d1aac5075b889b1b14caccc452dbe..5e06422cf9ada7b920e95aeb98ab2b671c6ff607 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -81,9 +81,9 @@ def riscv_vmv_x_s : SDNode<"RISCVISD::VMV_X_S", def riscv_read_vlenb : SDNode<"RISCVISD::READ_VLENB", SDTypeProfile<1, 0, [SDTCisVT<0, XLenVT>]>>; -// Operand that is allowed to be a register or a 5 bit immediate. -// This allows us to pick between VSETIVLI and VSETVLI opcodes using the same -// pseudo instructions. +// Operand that is allowed to be a register other than X0, a 5 bit unsigned +// immediate, or -1. -1 means VLMAX. This allows us to pick between VSETIVLI and +// VSETVLI opcodes using the same pseudo instructions. def AVL : RegisterOperand { let OperandNamespace = "RISCVOp"; let OperandType = "OPERAND_AVL"; diff --git a/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp b/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp index 02a8d5c18fe1a0eea3832ad4a828df83b860d76d..02ea5270823d8da26b6c509c76ce1a2a0a70c5f2 100644 --- a/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp +++ b/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp @@ -18,6 +18,101 @@ using namespace llvm; +static bool checkRegisters(Register FirstDest, const MachineInstr &SecondMI) { + if (!SecondMI.getOperand(1).isReg()) + return false; + + if (SecondMI.getOperand(1).getReg() != FirstDest) + return false; + + // If the input is virtual make sure this is the only user. + if (FirstDest.isVirtual()) { + auto &MRI = SecondMI.getMF()->getRegInfo(); + return MRI.hasOneNonDBGUse(FirstDest); + } + + return SecondMI.getOperand(0).getReg() == FirstDest; +} + +// Fuse load with add: +// add rd, rs1, rs2 +// ld rd, 0(rd) +static bool isLDADD(const MachineInstr *FirstMI, const MachineInstr &SecondMI) { + if (SecondMI.getOpcode() != RISCV::LD) + return false; + + if (!SecondMI.getOperand(2).isImm()) + return false; + + if (SecondMI.getOperand(2).getImm() != 0) + return false; + + // Given SecondMI, when FirstMI is unspecified, we must return + // if SecondMI may be part of a fused pair at all. + if (!FirstMI) + return true; + + if (FirstMI->getOpcode() != RISCV::ADD) + return true; + + return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI); +} + +// Fuse these patterns: +// +// slli rd, rs1, 32 +// srli rd, rd, x +// where 0 <= x <= 32 +// +// and +// +// slli rd, rs1, 48 +// srli rd, rd, x +static bool isShiftedZExt(const MachineInstr *FirstMI, + const MachineInstr &SecondMI) { + if (SecondMI.getOpcode() != RISCV::SRLI) + return false; + + if (!SecondMI.getOperand(2).isImm()) + return false; + + unsigned SRLIImm = SecondMI.getOperand(2).getImm(); + bool IsShiftBy48 = SRLIImm == 48; + if (SRLIImm > 32 && !IsShiftBy48) + return false; + + // Given SecondMI, when FirstMI is unspecified, we must return + // if SecondMI may be part of a fused pair at all. + if (!FirstMI) + return true; + + if (FirstMI->getOpcode() != RISCV::SLLI) + return false; + + unsigned SLLIImm = FirstMI->getOperand(2).getImm(); + if (IsShiftBy48 ? (SLLIImm != 48) : (SLLIImm != 32)) + return false; + + return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI); +} + +// Fuse AUIPC followed by ADDI +// auipc rd, imm20 +// addi rd, rd, imm12 +static bool isAUIPCADDI(const MachineInstr *FirstMI, + const MachineInstr &SecondMI) { + if (SecondMI.getOpcode() != RISCV::ADDI) + return false; + // Assume the 1st instr to be a wildcard if it is unspecified. + if (!FirstMI) + return true; + + if (FirstMI->getOpcode() != RISCV::AUIPC) + return false; + + return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI); +} + // Fuse LUI followed by ADDI or ADDIW. // rd = imm[31:0] which decomposes to // lui rd, imm[31:12] @@ -27,7 +122,6 @@ static bool isLUIADDI(const MachineInstr *FirstMI, if (SecondMI.getOpcode() != RISCV::ADDI && SecondMI.getOpcode() != RISCV::ADDIW) return false; - // Assume the 1st instr to be a wildcard if it is unspecified. if (!FirstMI) return true; @@ -35,21 +129,7 @@ static bool isLUIADDI(const MachineInstr *FirstMI, if (FirstMI->getOpcode() != RISCV::LUI) return false; - Register FirstDest = FirstMI->getOperand(0).getReg(); - - // Destination of LUI should be the ADDI(W) source register. - if (SecondMI.getOperand(1).getReg() != FirstDest) - return false; - - // If the input is virtual make sure this is the only user. - if (FirstDest.isVirtual()) { - auto &MRI = SecondMI.getMF()->getRegInfo(); - return MRI.hasOneNonDBGUse(FirstDest); - } - - // If the FirstMI destination is non-virtual, it should match the SecondMI - // destination. - return SecondMI.getOperand(0).getReg() == FirstDest; + return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI); } static bool shouldScheduleAdjacent(const TargetInstrInfo &TII, @@ -61,6 +141,15 @@ static bool shouldScheduleAdjacent(const TargetInstrInfo &TII, if (ST.hasLUIADDIFusion() && isLUIADDI(FirstMI, SecondMI)) return true; + if (ST.hasAUIPCADDIFusion() && isAUIPCADDI(FirstMI, SecondMI)) + return true; + + if (ST.hasShiftedZExtFusion() && isShiftedZExt(FirstMI, SecondMI)) + return true; + + if (ST.hasLDADDFusion() && isLDADD(FirstMI, SecondMI)) + return true; + return false; } diff --git a/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp b/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp index 79da67d06e1898d848c9390bcdcb06e22d933c8c..ff21fe1d406463762f0dbb4eac9522238a385339 100644 --- a/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp +++ b/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp @@ -84,9 +84,7 @@ struct RISCVMakeCompressibleOpt : public MachineFunctionPass { bool runOnMachineFunction(MachineFunction &Fn) override; - RISCVMakeCompressibleOpt() : MachineFunctionPass(ID) { - initializeRISCVMakeCompressibleOptPass(*PassRegistry::getPassRegistry()); - } + RISCVMakeCompressibleOpt() : MachineFunctionPass(ID) {} StringRef getPassName() const override { return RISCV_COMPRESS_INSTRS_NAME; } }; diff --git a/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp b/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp index 097e12a2cd59f0d448fc4de62fadbfbf2b3aa28d..ae46d5554d3505dd72b01e97acd224093c842bc3 100644 --- a/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp +++ b/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp @@ -19,7 +19,6 @@ #include "llvm/Support/Debug.h" #include "llvm/Target/TargetOptions.h" #include -#include using namespace llvm; #define DEBUG_TYPE "riscv-merge-base-offset" diff --git a/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp b/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp index 934a2a0f8e1afc9380264127ecb6b7c528e049b6..3c5462057b280743a7132385791d69e5bc637c37 100644 --- a/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp +++ b/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp @@ -22,9 +22,7 @@ namespace { struct RISCVMoveMerge : public MachineFunctionPass { static char ID; - RISCVMoveMerge() : MachineFunctionPass(ID) { - initializeRISCVMoveMergePass(*PassRegistry::getPassRegistry()); - } + RISCVMoveMerge() : MachineFunctionPass(ID) {} const RISCVInstrInfo *TII; const TargetRegisterInfo *TRI; diff --git a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp index b667a2b7a11bac565c622d2fbda98441a02ca55d..2c2b34bb5b779761145a1ad23ec00a4f02e182cc 100644 --- a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp +++ b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp @@ -53,9 +53,7 @@ class RISCVOptWInstrs : public MachineFunctionPass { public: static char ID; - RISCVOptWInstrs() : MachineFunctionPass(ID) { - initializeRISCVOptWInstrsPass(*PassRegistry::getPassRegistry()); - } + RISCVOptWInstrs() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; bool removeSExtWInstrs(MachineFunction &MF, const RISCVInstrInfo &TII, @@ -368,6 +366,18 @@ static bool isSignExtendingOpW(const MachineInstr &MI, return MI.getOperand(1).getReg() == RISCV::X0; case RISCV::PseudoAtomicLoadNand32: return true; + case RISCV::PseudoVMV_X_S_MF8: + case RISCV::PseudoVMV_X_S_MF4: + case RISCV::PseudoVMV_X_S_MF2: + case RISCV::PseudoVMV_X_S_M1: + case RISCV::PseudoVMV_X_S_M2: + case RISCV::PseudoVMV_X_S_M4: + case RISCV::PseudoVMV_X_S_M8: { + // vmv.x.s has at least 33 sign bits if log2(sew) <= 5. + int64_t Log2SEW = MI.getOperand(2).getImm(); + assert(Log2SEW >= 3 && Log2SEW <= 6 && "Unexpected Log2SEW"); + return Log2SEW <= 5; + } } return false; diff --git a/llvm/lib/Target/RISCV/RISCVProcessors.td b/llvm/lib/Target/RISCV/RISCVProcessors.td index 90ba99d3f845d38e56afffd66347a32943143c5b..58989fd716fa0eef005cb0afa1a1599203732932 100644 --- a/llvm/lib/Target/RISCV/RISCVProcessors.td +++ b/llvm/lib/Target/RISCV/RISCVProcessors.td @@ -254,7 +254,7 @@ def VENTANA_VEYRON_V1 : RISCVProcessorModel<"veyron-v1", FeatureStdExtZicbop, FeatureStdExtZicboz, FeatureVendorXVentanaCondOps], - [TuneVentanaVeyron]>; + [TuneVeyronFusions]>; def XIANGSHAN_NANHU : RISCVProcessorModel<"xiangshan-nanhu", NoSchedModel, diff --git a/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp index c2c795ec946a9d09dea187cdc7240612bdc77dd6..009dcf57f46da425ded3badb52b65549a5fdd757 100644 --- a/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp +++ b/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp @@ -22,9 +22,7 @@ namespace { struct RISCVPushPopOpt : public MachineFunctionPass { static char ID; - RISCVPushPopOpt() : MachineFunctionPass(ID) { - initializeRISCVPushPopOptPass(*PassRegistry::getPassRegistry()); - } + RISCVPushPopOpt() : MachineFunctionPass(ID) {} const RISCVInstrInfo *TII; const TargetRegisterInfo *TRI; diff --git a/llvm/lib/Target/RISCV/RISCVRVVInitUndef.cpp b/llvm/lib/Target/RISCV/RISCVRVVInitUndef.cpp index ff7718c9ef6d937cb998698d5d86c9b46e791859..735fc1350c0091559877f5a2aa3bfadee53c8ea3 100644 --- a/llvm/lib/Target/RISCV/RISCVRVVInitUndef.cpp +++ b/llvm/lib/Target/RISCV/RISCVRVVInitUndef.cpp @@ -65,9 +65,7 @@ class RISCVInitUndef : public MachineFunctionPass { public: static char ID; - RISCVInitUndef() : MachineFunctionPass(ID) { - initializeRISCVInitUndefPass(*PassRegistry::getPassRegistry()); - } + RISCVInitUndef() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; void getAnalysisUsage(AnalysisUsage &AU) const override { diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h index 52f00f1f099030cf8d6b65e1aa6488cab712603f..23d56cfa6e4e521e3889a5ecb2f568959cdccd74 100644 --- a/llvm/lib/Target/RISCV/RISCVSubtarget.h +++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h @@ -25,6 +25,7 @@ #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/DataLayout.h" #include "llvm/Target/TargetMachine.h" +#include #define GET_SUBTARGETINFO_HEADER #include "RISCVGenSubtargetInfo.inc" @@ -192,7 +193,10 @@ public: return UserReservedRegister[i]; } - bool hasMacroFusion() const { return hasLUIADDIFusion(); } + bool hasMacroFusion() const { + return hasLUIADDIFusion() || hasAUIPCADDIFusion() || + hasShiftedZExtFusion() || hasLDADDFusion(); + } // Vector codegen related methods. bool hasVInstructions() const { return HasStdExtZve32x; } diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp index ba13f8d2f4480f1e7ef2c12381c6bfe99ee6b9ff..3abdb6003659fa29a4245fd3a862e317066ec01c 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp @@ -213,13 +213,8 @@ RISCVTargetMachine::getSubtargetImpl(const Function &F) const { llvm::bit_floor((RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax); SmallString<512> Key; - Key += "RVVMin"; - Key += std::to_string(RVVBitsMin); - Key += "RVVMax"; - Key += std::to_string(RVVBitsMax); - Key += CPU; - Key += TuneCPU; - Key += FS; + raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax + << CPU << TuneCPU << FS; auto &I = SubtargetMap[Key]; if (!I) { // This needs to be done before we create a new subtarget since any @@ -549,7 +544,7 @@ void RISCVPassConfig::addMachineSSAOptimization() { if (EnableMachineCombiner) addPass(&MachineCombinerID); - if (TM->getTargetTriple().getArch() == Triple::riscv64) { + if (TM->getTargetTriple().isRISCV64()) { addPass(createRISCVOptWInstrsPass()); } } diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 000d68d4a79ff1afb50b9373cb312d840980d99e..ec62a819b00eedfbdb88be01661277faadab1827 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -505,7 +505,6 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) { if (!SI) continue; Type *ElTy = SI->getValueOperand()->getType(); - PointerType *PTy = cast(SI->getOperand(1)->getType()); if (ElTy->isAggregateType() || ElTy->isVectorTy()) AggrStores.insert(&I); } diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 0e3fcf4701e6227e96e25dfa0d1fe4a678b6a6f2..d450613196f3e4a4d1fe531eec36fc7c256cbac8 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -707,12 +707,7 @@ SPIRVType *SPIRVGlobalRegistry::createSPIRVType( // At the moment, all opaque pointers correspond to i8 element type. // TODO: change the implementation once opaque pointers are supported // in the SPIR-V specification. - if (PType->isOpaque()) - SpvElementType = getOrCreateSPIRVIntegerType(8, MIRBuilder); - else - SpvElementType = - findSPIRVType(PType->getNonOpaquePointerElementType(), MIRBuilder, - SPIRV::AccessQualifier::ReadWrite, EmitIR); + SpvElementType = getOrCreateSPIRVIntegerType(8, MIRBuilder); auto SC = addressSpaceToStorageClass(PType->getAddressSpace()); // Null pointer means we have a loop in type definitions, make and // return corresponding OpTypeForwardPointer. diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp index 6cb51d782919d50305e4c114e2ae0ee9cbcfe40e..0bd51436082d5f84bca42fca4cc1436b3a51cdbb 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp @@ -326,10 +326,9 @@ std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) { } const Type *getTypedPtrEltType(const Type *Ty) { - auto PType = dyn_cast(Ty); - if (!PType || PType->isOpaque()) - return Ty; - return PType->getNonOpaquePointerElementType(); + // TODO: This function requires updating following the opaque pointer + // migration. + return Ty; } bool hasBuiltinTypePrefix(StringRef Name) { diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp index 9e14f96b6caa0418c2ed5166c05f0b87b16fc01b..240f5396855c832438c7b92e0421286eb324bc41 100644 --- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp +++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp @@ -272,7 +272,8 @@ namespace { } bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override { + const MCValue &Target, + const MCSubtargetInfo *STI) override { if (Fixup.getKind() >= FirstLiteralRelocationKind) return true; switch ((Sparc::Fixups)Fixup.getKind()) { diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp index eafe41124897d1208c39eeba17692655c2c85b22..e32023ccedf5385729ea56aeded5a992b71b3887 100644 --- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp +++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp @@ -116,7 +116,8 @@ public: std::optional getFixupKind(StringRef Name) const override; const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, uint64_t Value, bool IsResolved, @@ -164,8 +165,9 @@ SystemZMCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const { } bool SystemZMCAsmBackend::shouldForceRelocation(const MCAssembler &, - const MCFixup &Fixup, - const MCValue &) { + const MCFixup &Fixup, + const MCValue &, + const MCSubtargetInfo *STI) { return Fixup.getKind() >= FirstLiteralRelocationKind; } diff --git a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp index 4cc69530db011ef896e9213ad0a03c1e7be44ed3..0d8d5451b8136c2b298b059ff95afa4bfc6553b7 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp @@ -1558,6 +1558,9 @@ void SystemZDAGToDAGISel::Select(SDNode *Node) { break; } } + // Don't split an XOR with -1 as LCGR/AGHI is more compact. + if (Opcode == ISD::XOR && Op1->isAllOnes()) + break; if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) { splitLargeImmediate(Opcode, Node, Node->getOperand(0), Val - uint32_t(Val), uint32_t(Val)); diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp index 873994c2e333bf53b37be10772c3b3910e096d87..f79787d4baa4d832ecf07616aad468fac7ed7360 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -872,13 +872,21 @@ bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const TargetLowering::AtomicExpansionKind SystemZTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const { - // TODO: expand them all here instead of in backend. - return (RMW->isFloatingPointOperation() || - RMW->getOperation() == AtomicRMWInst::UIncWrap || - RMW->getOperation() == AtomicRMWInst::UDecWrap || - RMW->getType()->isIntegerTy(128)) - ? AtomicExpansionKind::CmpXChg - : AtomicExpansionKind::None; + // Don't expand subword operations as they require special treatment. + if (RMW->getType()->isIntegerTy(8) || RMW->getType()->isIntegerTy(16)) + return AtomicExpansionKind::None; + + // Don't expand if there is a target instruction available. + if (Subtarget.hasInterlockedAccess1() && + (RMW->getType()->isIntegerTy(32) || RMW->getType()->isIntegerTy(64)) && + (RMW->getOperation() == AtomicRMWInst::BinOp::Add || + RMW->getOperation() == AtomicRMWInst::BinOp::Sub || + RMW->getOperation() == AtomicRMWInst::BinOp::And || + RMW->getOperation() == AtomicRMWInst::BinOp::Or || + RMW->getOperation() == AtomicRMWInst::BinOp::Xor)) + return AtomicExpansionKind::None; + + return AtomicExpansionKind::CmpXChg; } bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const { @@ -4350,6 +4358,31 @@ SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op, return Chain; } +// Prepare for a Compare And Swap for a subword operation. This needs to be +// done in memory with 4 bytes at natural alignment. +static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL, + SDValue &AlignedAddr, SDValue &BitShift, + SDValue &NegBitShift) { + EVT PtrVT = Addr.getValueType(); + EVT WideVT = MVT::i32; + + // Get the address of the containing word. + AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, + DAG.getConstant(-4, DL, PtrVT)); + + // Get the number of bits that the word must be rotated left in order + // to bring the field to the top bits of a GR32. + BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, + DAG.getConstant(3, DL, PtrVT)); + BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); + + // Get the complementing shift amount, for rotating a field in the top + // bits back to its proper position. + NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, + DAG.getConstant(0, DL, WideVT), BitShift); + +} + // Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first // two into the fullword ATOMIC_LOADW_* operation given by Opcode. SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op, @@ -4357,7 +4390,7 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op, unsigned Opcode) const { auto *Node = cast(Op.getNode()); - // 32-bit operations need no code outside the main loop. + // 32-bit operations need no special handling. EVT NarrowVT = Node->getMemoryVT(); EVT WideVT = MVT::i32; if (NarrowVT == WideVT) @@ -4369,7 +4402,6 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op, SDValue Src2 = Node->getVal(); MachineMemOperand *MMO = Node->getMemOperand(); SDLoc DL(Node); - EVT PtrVT = Addr.getValueType(); // Convert atomic subtracts of constants into additions. if (Opcode == SystemZISD::ATOMIC_LOADW_SUB) @@ -4378,20 +4410,8 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op, Src2 = DAG.getConstant(-Const->getSExtValue(), DL, Src2.getValueType()); } - // Get the address of the containing word. - SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, - DAG.getConstant(-4, DL, PtrVT)); - - // Get the number of bits that the word must be rotated left in order - // to bring the field to the top bits of a GR32. - SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, - DAG.getConstant(3, DL, PtrVT)); - BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); - - // Get the complementing shift amount, for rotating a field in the top - // bits back to its proper position. - SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, - DAG.getConstant(0, DL, WideVT), BitShift); + SDValue AlignedAddr, BitShift, NegBitShift; + getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift); // Extend the source operand to 32 bits and prepare it for the inner loop. // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other @@ -4423,38 +4443,24 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op, return DAG.getMergeValues(RetOps, DL); } -// Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations -// into ATOMIC_LOADW_SUBs and decide whether to convert 32- and 64-bit -// operations into additions. +// Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations into +// ATOMIC_LOADW_SUBs and convert 32- and 64-bit operations into additions. SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const { auto *Node = cast(Op.getNode()); EVT MemVT = Node->getMemoryVT(); if (MemVT == MVT::i32 || MemVT == MVT::i64) { - // A full-width operation. + // A full-width operation: negate and use LAA(G). assert(Op.getValueType() == MemVT && "Mismatched VTs"); + assert(Subtarget.hasInterlockedAccess1() && + "Should have been expanded by AtomicExpand pass."); SDValue Src2 = Node->getVal(); - SDValue NegSrc2; SDLoc DL(Src2); - - if (auto *Op2 = dyn_cast(Src2)) { - // Use an addition if the operand is constant and either LAA(G) is - // available or the negative value is in the range of A(G)FHI. - int64_t Value = (-Op2->getAPIntValue()).getSExtValue(); - if (isInt<32>(Value) || Subtarget.hasInterlockedAccess1()) - NegSrc2 = DAG.getConstant(Value, DL, MemVT); - } else if (Subtarget.hasInterlockedAccess1()) - // Use LAA(G) if available. - NegSrc2 = DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT), - Src2); - - if (NegSrc2.getNode()) - return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT, - Node->getChain(), Node->getBasePtr(), NegSrc2, - Node->getMemOperand()); - - // Use the node as-is. - return Op; + SDValue NegSrc2 = + DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT), Src2); + return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT, + Node->getChain(), Node->getBasePtr(), NegSrc2, + Node->getMemOperand()); } return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB); @@ -4492,22 +4498,9 @@ SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op, // Convert 8-bit and 16-bit compare and swap to a loop, implemented // via a fullword ATOMIC_CMP_SWAPW operation. int64_t BitSize = NarrowVT.getSizeInBits(); - EVT PtrVT = Addr.getValueType(); - - // Get the address of the containing word. - SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, - DAG.getConstant(-4, DL, PtrVT)); - // Get the number of bits that the word must be rotated left in order - // to bring the field to the top bits of a GR32. - SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, - DAG.getConstant(3, DL, PtrVT)); - BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); - - // Get the complementing shift amount, for rotating a field in the top - // bits back to its proper position. - SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, - DAG.getConstant(0, DL, WideVT), BitShift); + SDValue AlignedAddr, BitShift, NegBitShift; + getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift); // Construct the ATOMIC_CMP_SWAPW node. SDVTList VTList = DAG.getVTList(WideVT, MVT::i32, MVT::Other); @@ -7951,20 +7944,17 @@ MachineBasicBlock *SystemZTargetLowering::emitCondStore(MachineInstr &MI, return JoinMBB; } -// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_LOAD{,W}_* -// or ATOMIC_SWAP{,W} instruction MI. BinOpcode is the instruction that -// performs the binary operation elided by "*", or 0 for ATOMIC_SWAP{,W}. -// BitSize is the width of the field in bits, or 0 if this is a partword -// ATOMIC_LOADW_* or ATOMIC_SWAPW instruction, in which case the bitsize -// is one of the operands. Invert says whether the field should be -// inverted after performing BinOpcode (e.g. for NAND). +// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_LOADW_* or +// ATOMIC_SWAPW instruction MI. BinOpcode is the instruction that performs +// the binary operation elided by "*", or 0 for ATOMIC_SWAPW. Invert says +// whether the field should be inverted after performing BinOpcode (e.g. for +// NAND). MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary( MachineInstr &MI, MachineBasicBlock *MBB, unsigned BinOpcode, - unsigned BitSize, bool Invert) const { + bool Invert) const { MachineFunction &MF = *MBB->getParent(); const SystemZInstrInfo *TII = Subtarget.getInstrInfo(); MachineRegisterInfo &MRI = MF.getRegInfo(); - bool IsSubWord = (BitSize < 32); // Extract the operands. Base can be a register or a frame index. // Src2 can be a register or immediate. @@ -7972,31 +7962,22 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary( MachineOperand Base = earlyUseOperand(MI.getOperand(1)); int64_t Disp = MI.getOperand(2).getImm(); MachineOperand Src2 = earlyUseOperand(MI.getOperand(3)); - Register BitShift = IsSubWord ? MI.getOperand(4).getReg() : Register(); - Register NegBitShift = IsSubWord ? MI.getOperand(5).getReg() : Register(); + Register BitShift = MI.getOperand(4).getReg(); + Register NegBitShift = MI.getOperand(5).getReg(); + unsigned BitSize = MI.getOperand(6).getImm(); DebugLoc DL = MI.getDebugLoc(); - if (IsSubWord) - BitSize = MI.getOperand(6).getImm(); - - // Subword operations use 32-bit registers. - const TargetRegisterClass *RC = (BitSize <= 32 ? - &SystemZ::GR32BitRegClass : - &SystemZ::GR64BitRegClass); - unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; - unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; // Get the right opcodes for the displacement. - LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); - CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); + unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); + unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); assert(LOpcode && CSOpcode && "Displacement out of range"); // Create virtual registers for temporary results. - Register OrigVal = MRI.createVirtualRegister(RC); - Register OldVal = MRI.createVirtualRegister(RC); - Register NewVal = (BinOpcode || IsSubWord ? - MRI.createVirtualRegister(RC) : Src2.getReg()); - Register RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); - Register RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); + Register OrigVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register OldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register NewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register RotatedOldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register RotatedNewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); // Insert a basic block for the main loop. MachineBasicBlock *StartMBB = MBB; @@ -8023,39 +8004,28 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary( BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) .addReg(OrigVal).addMBB(StartMBB) .addReg(Dest).addMBB(LoopMBB); - if (IsSubWord) - BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) - .addReg(OldVal).addReg(BitShift).addImm(0); + BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) + .addReg(OldVal).addReg(BitShift).addImm(0); if (Invert) { // Perform the operation normally and then invert every bit of the field. - Register Tmp = MRI.createVirtualRegister(RC); + Register Tmp = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); BuildMI(MBB, DL, TII->get(BinOpcode), Tmp).addReg(RotatedOldVal).add(Src2); - if (BitSize <= 32) - // XILF with the upper BitSize bits set. - BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal) - .addReg(Tmp).addImm(-1U << (32 - BitSize)); - else { - // Use LCGR and add -1 to the result, which is more compact than - // an XILF, XILH pair. - Register Tmp2 = MRI.createVirtualRegister(RC); - BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp); - BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal) - .addReg(Tmp2).addImm(-1); - } + // XILF with the upper BitSize bits set. + BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal) + .addReg(Tmp).addImm(-1U << (32 - BitSize)); } else if (BinOpcode) // A simply binary operation. BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal) .addReg(RotatedOldVal) .add(Src2); - else if (IsSubWord) + else // Use RISBG to rotate Src2 into position and use it to replace the // field in RotatedOldVal. BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal) .addReg(RotatedOldVal).addReg(Src2.getReg()) .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize); - if (IsSubWord) - BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) - .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); + BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) + .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); BuildMI(MBB, DL, TII->get(CSOpcode), Dest) .addReg(OldVal) .addReg(NewVal) @@ -8070,50 +8040,40 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary( return DoneMBB; } -// Implement EmitInstrWithCustomInserter for pseudo -// ATOMIC_LOAD{,W}_{,U}{MIN,MAX} instruction MI. CompareOpcode is the +// Implement EmitInstrWithCustomInserter for subword pseudo +// ATOMIC_LOADW_{,U}{MIN,MAX} instruction MI. CompareOpcode is the // instruction that should be used to compare the current field with the // minimum or maximum value. KeepOldMask is the BRC condition-code mask -// for when the current field should be kept. BitSize is the width of -// the field in bits, or 0 if this is a partword ATOMIC_LOADW_* instruction. +// for when the current field should be kept. MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax( MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode, - unsigned KeepOldMask, unsigned BitSize) const { + unsigned KeepOldMask) const { MachineFunction &MF = *MBB->getParent(); const SystemZInstrInfo *TII = Subtarget.getInstrInfo(); MachineRegisterInfo &MRI = MF.getRegInfo(); - bool IsSubWord = (BitSize < 32); // Extract the operands. Base can be a register or a frame index. Register Dest = MI.getOperand(0).getReg(); MachineOperand Base = earlyUseOperand(MI.getOperand(1)); int64_t Disp = MI.getOperand(2).getImm(); Register Src2 = MI.getOperand(3).getReg(); - Register BitShift = (IsSubWord ? MI.getOperand(4).getReg() : Register()); - Register NegBitShift = (IsSubWord ? MI.getOperand(5).getReg() : Register()); + Register BitShift = MI.getOperand(4).getReg(); + Register NegBitShift = MI.getOperand(5).getReg(); + unsigned BitSize = MI.getOperand(6).getImm(); DebugLoc DL = MI.getDebugLoc(); - if (IsSubWord) - BitSize = MI.getOperand(6).getImm(); - - // Subword operations use 32-bit registers. - const TargetRegisterClass *RC = (BitSize <= 32 ? - &SystemZ::GR32BitRegClass : - &SystemZ::GR64BitRegClass); - unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; - unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; // Get the right opcodes for the displacement. - LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); - CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); + unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); + unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); assert(LOpcode && CSOpcode && "Displacement out of range"); // Create virtual registers for temporary results. - Register OrigVal = MRI.createVirtualRegister(RC); - Register OldVal = MRI.createVirtualRegister(RC); - Register NewVal = MRI.createVirtualRegister(RC); - Register RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); - Register RotatedAltVal = (IsSubWord ? MRI.createVirtualRegister(RC) : Src2); - Register RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); + Register OrigVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register OldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register NewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register RotatedOldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register RotatedAltVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register RotatedNewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); // Insert 3 basic blocks for the loop. MachineBasicBlock *StartMBB = MBB; @@ -8139,9 +8099,8 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax( BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) .addReg(OrigVal).addMBB(StartMBB) .addReg(Dest).addMBB(UpdateMBB); - if (IsSubWord) - BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) - .addReg(OldVal).addReg(BitShift).addImm(0); + BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) + .addReg(OldVal).addReg(BitShift).addImm(0); BuildMI(MBB, DL, TII->get(CompareOpcode)) .addReg(RotatedOldVal).addReg(Src2); BuildMI(MBB, DL, TII->get(SystemZ::BRC)) @@ -8153,10 +8112,9 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax( // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0 // # fall through to UpdateMBB MBB = UseAltMBB; - if (IsSubWord) - BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) - .addReg(RotatedOldVal).addReg(Src2) - .addImm(32).addImm(31 + BitSize).addImm(0); + BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) + .addReg(RotatedOldVal).addReg(Src2) + .addImm(32).addImm(31 + BitSize).addImm(0); MBB->addSuccessor(UpdateMBB); // UpdateMBB: @@ -8170,9 +8128,8 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax( BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal) .addReg(RotatedOldVal).addMBB(LoopMBB) .addReg(RotatedAltVal).addMBB(UseAltMBB); - if (IsSubWord) - BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) - .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); + BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) + .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); BuildMI(MBB, DL, TII->get(CSOpcode), Dest) .addReg(OldVal) .addReg(NewVal) @@ -8187,7 +8144,7 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax( return DoneMBB; } -// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_CMP_SWAPW +// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_CMP_SWAPW // instruction MI. MachineBasicBlock * SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI, @@ -9004,171 +8961,44 @@ MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter( return emitExt128(MI, MBB, true); case SystemZ::ATOMIC_SWAPW: - return emitAtomicLoadBinary(MI, MBB, 0, 0); - case SystemZ::ATOMIC_SWAP_32: - return emitAtomicLoadBinary(MI, MBB, 0, 32); - case SystemZ::ATOMIC_SWAP_64: - return emitAtomicLoadBinary(MI, MBB, 0, 64); + return emitAtomicLoadBinary(MI, MBB, 0); case SystemZ::ATOMIC_LOADW_AR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 0); + return emitAtomicLoadBinary(MI, MBB, SystemZ::AR); case SystemZ::ATOMIC_LOADW_AFI: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 0); - case SystemZ::ATOMIC_LOAD_AR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 32); - case SystemZ::ATOMIC_LOAD_AHI: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AHI, 32); - case SystemZ::ATOMIC_LOAD_AFI: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 32); - case SystemZ::ATOMIC_LOAD_AGR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AGR, 64); - case SystemZ::ATOMIC_LOAD_AGHI: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AGHI, 64); - case SystemZ::ATOMIC_LOAD_AGFI: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AGFI, 64); + return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI); case SystemZ::ATOMIC_LOADW_SR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 0); - case SystemZ::ATOMIC_LOAD_SR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 32); - case SystemZ::ATOMIC_LOAD_SGR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::SGR, 64); + return emitAtomicLoadBinary(MI, MBB, SystemZ::SR); case SystemZ::ATOMIC_LOADW_NR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0); + return emitAtomicLoadBinary(MI, MBB, SystemZ::NR); case SystemZ::ATOMIC_LOADW_NILH: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0); - case SystemZ::ATOMIC_LOAD_NR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32); - case SystemZ::ATOMIC_LOAD_NILL: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32); - case SystemZ::ATOMIC_LOAD_NILH: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32); - case SystemZ::ATOMIC_LOAD_NILF: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32); - case SystemZ::ATOMIC_LOAD_NGR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64); - case SystemZ::ATOMIC_LOAD_NILL64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64); - case SystemZ::ATOMIC_LOAD_NILH64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64); - case SystemZ::ATOMIC_LOAD_NIHL64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64); - case SystemZ::ATOMIC_LOAD_NIHH64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64); - case SystemZ::ATOMIC_LOAD_NILF64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64); - case SystemZ::ATOMIC_LOAD_NIHF64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64); + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH); case SystemZ::ATOMIC_LOADW_OR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 0); + return emitAtomicLoadBinary(MI, MBB, SystemZ::OR); case SystemZ::ATOMIC_LOADW_OILH: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 0); - case SystemZ::ATOMIC_LOAD_OR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 32); - case SystemZ::ATOMIC_LOAD_OILL: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL, 32); - case SystemZ::ATOMIC_LOAD_OILH: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 32); - case SystemZ::ATOMIC_LOAD_OILF: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF, 32); - case SystemZ::ATOMIC_LOAD_OGR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OGR, 64); - case SystemZ::ATOMIC_LOAD_OILL64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL64, 64); - case SystemZ::ATOMIC_LOAD_OILH64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH64, 64); - case SystemZ::ATOMIC_LOAD_OIHL64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHL64, 64); - case SystemZ::ATOMIC_LOAD_OIHH64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHH64, 64); - case SystemZ::ATOMIC_LOAD_OILF64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF64, 64); - case SystemZ::ATOMIC_LOAD_OIHF64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHF64, 64); + return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH); case SystemZ::ATOMIC_LOADW_XR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 0); + return emitAtomicLoadBinary(MI, MBB, SystemZ::XR); case SystemZ::ATOMIC_LOADW_XILF: - return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 0); - case SystemZ::ATOMIC_LOAD_XR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 32); - case SystemZ::ATOMIC_LOAD_XILF: - return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 32); - case SystemZ::ATOMIC_LOAD_XGR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::XGR, 64); - case SystemZ::ATOMIC_LOAD_XILF64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF64, 64); - case SystemZ::ATOMIC_LOAD_XIHF64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::XIHF64, 64); + return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF); case SystemZ::ATOMIC_LOADW_NRi: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0, true); + return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, true); case SystemZ::ATOMIC_LOADW_NILHi: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0, true); - case SystemZ::ATOMIC_LOAD_NRi: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32, true); - case SystemZ::ATOMIC_LOAD_NILLi: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32, true); - case SystemZ::ATOMIC_LOAD_NILHi: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32, true); - case SystemZ::ATOMIC_LOAD_NILFi: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32, true); - case SystemZ::ATOMIC_LOAD_NGRi: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64, true); - case SystemZ::ATOMIC_LOAD_NILL64i: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64, true); - case SystemZ::ATOMIC_LOAD_NILH64i: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64, true); - case SystemZ::ATOMIC_LOAD_NIHL64i: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64, true); - case SystemZ::ATOMIC_LOAD_NIHH64i: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64, true); - case SystemZ::ATOMIC_LOAD_NILF64i: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64, true); - case SystemZ::ATOMIC_LOAD_NIHF64i: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64, true); + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, true); case SystemZ::ATOMIC_LOADW_MIN: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, - SystemZ::CCMASK_CMP_LE, 0); - case SystemZ::ATOMIC_LOAD_MIN_32: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, - SystemZ::CCMASK_CMP_LE, 32); - case SystemZ::ATOMIC_LOAD_MIN_64: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, - SystemZ::CCMASK_CMP_LE, 64); - + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, SystemZ::CCMASK_CMP_LE); case SystemZ::ATOMIC_LOADW_MAX: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, - SystemZ::CCMASK_CMP_GE, 0); - case SystemZ::ATOMIC_LOAD_MAX_32: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, - SystemZ::CCMASK_CMP_GE, 32); - case SystemZ::ATOMIC_LOAD_MAX_64: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, - SystemZ::CCMASK_CMP_GE, 64); - + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, SystemZ::CCMASK_CMP_GE); case SystemZ::ATOMIC_LOADW_UMIN: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, - SystemZ::CCMASK_CMP_LE, 0); - case SystemZ::ATOMIC_LOAD_UMIN_32: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, - SystemZ::CCMASK_CMP_LE, 32); - case SystemZ::ATOMIC_LOAD_UMIN_64: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, - SystemZ::CCMASK_CMP_LE, 64); - + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, SystemZ::CCMASK_CMP_LE); case SystemZ::ATOMIC_LOADW_UMAX: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, - SystemZ::CCMASK_CMP_GE, 0); - case SystemZ::ATOMIC_LOAD_UMAX_32: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, - SystemZ::CCMASK_CMP_GE, 32); - case SystemZ::ATOMIC_LOAD_UMAX_64: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, - SystemZ::CCMASK_CMP_GE, 64); + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, SystemZ::CCMASK_CMP_GE); case SystemZ::ATOMIC_CMP_SWAPW: return emitAtomicCmpSwapW(MI, MBB); diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h index 40fe433f816fa6a2cb1302c1b454bfaac7bc8249..1e2887cff8164b231a0466ae9858d5cd1a24449a 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h @@ -748,13 +748,12 @@ private: bool ClearEven) const; MachineBasicBlock *emitAtomicLoadBinary(MachineInstr &MI, MachineBasicBlock *BB, - unsigned BinOpcode, unsigned BitSize, + unsigned BinOpcode, bool Invert = false) const; MachineBasicBlock *emitAtomicLoadMinMax(MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode, - unsigned KeepOldMask, - unsigned BitSize) const; + unsigned KeepOldMask) const; MachineBasicBlock *emitAtomicCmpSwapW(MachineInstr &MI, MachineBasicBlock *BB) const; MachineBasicBlock *emitMemMemWrapper(MachineInstr &MI, MachineBasicBlock *BB, diff --git a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td index a25719f80ad0771b872dfdf142a55d4c5187bdb6..2e5ff4a1df673b84188f210ba5d4fbd2fc8d285e 100644 --- a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td +++ b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td @@ -5327,30 +5327,6 @@ multiclass CondStores - : Pseudo<(outs cls:$dst), (ins bdaddr20only:$ptr, operand:$src2), - [(set cls:$dst, (operator bdaddr20only:$ptr, pat))]> { - let Defs = [CC]; - let Has20BitOffset = 1; - let mayLoad = 1; - let mayStore = 1; - let usesCustomInserter = 1; - let hasNoSchedulingInfo = 1; -} - -// Specializations of AtomicLoadWBinary. -class AtomicLoadBinaryReg32 - : AtomicLoadBinary; -class AtomicLoadBinaryImm32 - : AtomicLoadBinary; -class AtomicLoadBinaryReg64 - : AtomicLoadBinary; -class AtomicLoadBinaryImm64 - : AtomicLoadBinary; - // OPERATOR is ATOMIC_SWAPW or an ATOMIC_LOADW_* operation. PAT and OPERAND // describe the second (non-memory) operand. class AtomicLoadWBinary; -def ATOMIC_SWAP_32 : AtomicLoadBinaryReg32; -def ATOMIC_SWAP_64 : AtomicLoadBinaryReg64; def ATOMIC_LOADW_AR : AtomicLoadWBinaryReg; def ATOMIC_LOADW_AFI : AtomicLoadWBinaryImm; -let Predicates = [FeatureNoInterlockedAccess1] in { - def ATOMIC_LOAD_AR : AtomicLoadBinaryReg32; - def ATOMIC_LOAD_AHI : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_AFI : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_AGR : AtomicLoadBinaryReg64; - def ATOMIC_LOAD_AGHI : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_AGFI : AtomicLoadBinaryImm64; -} def ATOMIC_LOADW_SR : AtomicLoadWBinaryReg; -def ATOMIC_LOAD_SR : AtomicLoadBinaryReg32; -def ATOMIC_LOAD_SGR : AtomicLoadBinaryReg64; def ATOMIC_LOADW_NR : AtomicLoadWBinaryReg; def ATOMIC_LOADW_NILH : AtomicLoadWBinaryImm; -let Predicates = [FeatureNoInterlockedAccess1] in { - def ATOMIC_LOAD_NR : AtomicLoadBinaryReg32; - def ATOMIC_LOAD_NILL : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_NILH : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_NILF : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_NGR : AtomicLoadBinaryReg64; - def ATOMIC_LOAD_NILL64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_NILH64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_NIHL64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_NIHH64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_NILF64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_NIHF64 : AtomicLoadBinaryImm64; -} def ATOMIC_LOADW_OR : AtomicLoadWBinaryReg; def ATOMIC_LOADW_OILH : AtomicLoadWBinaryImm; -let Predicates = [FeatureNoInterlockedAccess1] in { - def ATOMIC_LOAD_OR : AtomicLoadBinaryReg32; - def ATOMIC_LOAD_OILL : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_OILH : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_OILF : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_OGR : AtomicLoadBinaryReg64; - def ATOMIC_LOAD_OILL64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_OILH64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_OIHL64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_OIHH64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_OILF64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_OIHF64 : AtomicLoadBinaryImm64; -} def ATOMIC_LOADW_XR : AtomicLoadWBinaryReg; def ATOMIC_LOADW_XILF : AtomicLoadWBinaryImm; -let Predicates = [FeatureNoInterlockedAccess1] in { - def ATOMIC_LOAD_XR : AtomicLoadBinaryReg32; - def ATOMIC_LOAD_XILF : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_XGR : AtomicLoadBinaryReg64; - def ATOMIC_LOAD_XILF64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_XIHF64 : AtomicLoadBinaryImm64; -} def ATOMIC_LOADW_NRi : AtomicLoadWBinaryReg; def ATOMIC_LOADW_NILHi : AtomicLoadWBinaryImm; -def ATOMIC_LOAD_NRi : AtomicLoadBinaryReg32; -def ATOMIC_LOAD_NILLi : AtomicLoadBinaryImm32; -def ATOMIC_LOAD_NILHi : AtomicLoadBinaryImm32; -def ATOMIC_LOAD_NILFi : AtomicLoadBinaryImm32; -def ATOMIC_LOAD_NGRi : AtomicLoadBinaryReg64; -def ATOMIC_LOAD_NILL64i : AtomicLoadBinaryImm64; -def ATOMIC_LOAD_NILH64i : AtomicLoadBinaryImm64; -def ATOMIC_LOAD_NIHL64i : AtomicLoadBinaryImm64; -def ATOMIC_LOAD_NIHH64i : AtomicLoadBinaryImm64; -def ATOMIC_LOAD_NILF64i : AtomicLoadBinaryImm64; -def ATOMIC_LOAD_NIHF64i : AtomicLoadBinaryImm64; def ATOMIC_LOADW_MIN : AtomicLoadWBinaryReg; -def ATOMIC_LOAD_MIN_32 : AtomicLoadBinaryReg32; -def ATOMIC_LOAD_MIN_64 : AtomicLoadBinaryReg64; - def ATOMIC_LOADW_MAX : AtomicLoadWBinaryReg; -def ATOMIC_LOAD_MAX_32 : AtomicLoadBinaryReg32; -def ATOMIC_LOAD_MAX_64 : AtomicLoadBinaryReg64; - def ATOMIC_LOADW_UMIN : AtomicLoadWBinaryReg; -def ATOMIC_LOAD_UMIN_32 : AtomicLoadBinaryReg32; -def ATOMIC_LOAD_UMIN_64 : AtomicLoadBinaryReg64; - def ATOMIC_LOADW_UMAX : AtomicLoadWBinaryReg; -def ATOMIC_LOAD_UMAX_32 : AtomicLoadBinaryReg32; -def ATOMIC_LOAD_UMAX_64 : AtomicLoadBinaryReg64; def ATOMIC_CMP_SWAPW : Pseudo<(outs GR32:$dst), (ins bdaddr20only:$addr, GR32:$cmp, GR32:$swap, @@ -2346,6 +2263,10 @@ let isCodeGenOnly = 1, hasSideEffects = 1 in { def : Pat<(and (xor GR64:$x, (i64 -1)), GR64:$y), (XGR GR64:$y, (NGR GR64:$y, GR64:$x))>; +// Use LCGR/AGHI for i64 xor with -1. +def : Pat<(xor GR64:$x, (i64 -1)), + (AGHI (LCGR GR64:$x), (i64 -1))>; + // Shift/rotate instructions only use the last 6 bits of the second operand // register, so we can safely use NILL (16 fewer bits than NILF) to only AND the // last 16 bits. diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp index 2773a7aabab740a4fad9244c3ca884d0bf0c9e62..05f7c00bdc14fb42c2a8f6d982877bcdf5eb87a9 100644 --- a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp +++ b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp @@ -132,7 +132,8 @@ public: } bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override { + const MCValue &Target, + const MCSubtargetInfo *STI) override { switch ((VE::Fixups)Fixup.getKind()) { default: return false; diff --git a/llvm/lib/Target/X86/MCA/X86CustomBehaviour.h b/llvm/lib/Target/X86/MCA/X86CustomBehaviour.h index 3cb5edeee5b5e269ae6fa193124f15785f48619d..4a83ba848dd88743b50d1ff511300fdec28ead2d 100644 --- a/llvm/lib/Target/X86/MCA/X86CustomBehaviour.h +++ b/llvm/lib/Target/X86/MCA/X86CustomBehaviour.h @@ -17,7 +17,6 @@ #ifndef LLVM_LIB_TARGET_X86_MCA_X86CUSTOMBEHAVIOUR_H #define LLVM_LIB_TARGET_X86_MCA_X86CUSTOMBEHAVIOUR_H -#include "llvm/ADT/SmallVector.h" #include "llvm/MCA/CustomBehaviour.h" #include "llvm/TargetParser/TargetParser.h" diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp index e01ce4f43143bd5e39f3823c14f32d271b046398..99dc9797f6df92c09e7016eff6a8c5107667bb20 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp @@ -173,7 +173,8 @@ public: const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, @@ -645,8 +646,8 @@ const MCFixupKindInfo &X86AsmBackend::getFixupKindInfo(MCFixupKind Kind) const { } bool X86AsmBackend::shouldForceRelocation(const MCAssembler &, - const MCFixup &Fixup, - const MCValue &) { + const MCFixup &Fixup, const MCValue &, + const MCSubtargetInfo *STI) { return Fixup.getKind() >= FirstLiteralRelocationKind; } diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp index ee82faebb57e6ce6ba50749215667b9d943101af..20b37d5a99902bed455c09fc8f2c582010cc8ead 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp @@ -1285,8 +1285,8 @@ bool llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, Src2Name = getRegName(MI->getOperand(2).getReg()); break; - case X86::VBROADCASTF128: - case X86::VBROADCASTI128: + case X86::VBROADCASTF128rm: + case X86::VBROADCASTI128rm: CASE_AVX512_INS_COMMON(BROADCASTF64X2, Z128, rm) CASE_AVX512_INS_COMMON(BROADCASTI64X2, Z128, rm) DecodeSubVectorBroadcast(4, 2, ShuffleMask); diff --git a/llvm/lib/Target/X86/X86FixupVectorConstants.cpp b/llvm/lib/Target/X86/X86FixupVectorConstants.cpp index 99e92bbcf996d2a1db86b1308145ffb7dc167283..5cc3b26dddafa8473e82be8b3f9a4bef60272e4d 100644 --- a/llvm/lib/Target/X86/X86FixupVectorConstants.cpp +++ b/llvm/lib/Target/X86/X86FixupVectorConstants.cpp @@ -190,12 +190,13 @@ static Constant *rebuildSplatableConstant(const Constant *C, Type *SclTy = OriginalType->getScalarType(); unsigned NumSclBits = SclTy->getPrimitiveSizeInBits(); NumSclBits = std::min(NumSclBits, SplatBitWidth); + LLVMContext &Ctx = OriginalType->getContext(); if (NumSclBits == 8) { SmallVector RawBits; for (unsigned I = 0; I != SplatBitWidth; I += 8) RawBits.push_back(Splat->extractBits(8, I).getZExtValue()); - return ConstantDataVector::get(OriginalType->getContext(), RawBits); + return ConstantDataVector::get(Ctx, RawBits); } if (NumSclBits == 16) { @@ -204,25 +205,25 @@ static Constant *rebuildSplatableConstant(const Constant *C, RawBits.push_back(Splat->extractBits(16, I).getZExtValue()); if (SclTy->is16bitFPTy()) return ConstantDataVector::getFP(SclTy, RawBits); - return ConstantDataVector::get(OriginalType->getContext(), RawBits); + return ConstantDataVector::get(Ctx, RawBits); } if (NumSclBits == 32) { SmallVector RawBits; for (unsigned I = 0; I != SplatBitWidth; I += 32) RawBits.push_back(Splat->extractBits(32, I).getZExtValue()); - if (SclTy->isFloatTy()) - return ConstantDataVector::getFP(SclTy, RawBits); - return ConstantDataVector::get(OriginalType->getContext(), RawBits); + if (SclTy->isFloatingPointTy()) + return ConstantDataVector::getFP(Type::getFloatTy(Ctx), RawBits); + return ConstantDataVector::get(Ctx, RawBits); } // Fallback to i64 / double. SmallVector RawBits; for (unsigned I = 0; I != SplatBitWidth; I += 64) RawBits.push_back(Splat->extractBits(64, I).getZExtValue()); - if (SclTy->isDoubleTy()) - return ConstantDataVector::getFP(SclTy, RawBits); - return ConstantDataVector::get(OriginalType->getContext(), RawBits); + if (SclTy->isFloatingPointTy()) + return ConstantDataVector::getFP(Type::getDoubleTy(Ctx), RawBits); + return ConstantDataVector::get(Ctx, RawBits); } bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF, @@ -285,7 +286,7 @@ bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF, case X86::VMOVAPSYrm: case X86::VMOVUPDYrm: case X86::VMOVUPSYrm: - return ConvertToBroadcast(0, X86::VBROADCASTF128, X86::VBROADCASTSDYrm, + return ConvertToBroadcast(0, X86::VBROADCASTF128rm, X86::VBROADCASTSDYrm, X86::VBROADCASTSSYrm, 0, 0, 1); case X86::VMOVAPDZ128rm: case X86::VMOVAPSZ128rm: @@ -318,7 +319,7 @@ bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF, case X86::VMOVDQAYrm: case X86::VMOVDQUYrm: return ConvertToBroadcast( - 0, HasAVX2 ? X86::VBROADCASTI128 : X86::VBROADCASTF128, + 0, HasAVX2 ? X86::VBROADCASTI128rm : X86::VBROADCASTF128rm, HasAVX2 ? X86::VPBROADCASTQYrm : X86::VBROADCASTSDYrm, HasAVX2 ? X86::VPBROADCASTDYrm : X86::VBROADCASTSSYrm, HasAVX2 ? X86::VPBROADCASTWYrm : 0, HasAVX2 ? X86::VPBROADCASTBYrm : 0, diff --git a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp index d2e3c708f59d0ac465140be4ebdf5e4a303ff84d..b13bf361ab79b5d42a6f62957694d9fab400809d 100644 --- a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp +++ b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp @@ -24,7 +24,6 @@ #include "X86InstrBuilder.h" #include "X86InstrInfo.h" #include "X86Subtarget.h" -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/ScopeExit.h" diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 4fca5afb46dd2b0a0c2de47f759fcc7cffb4af6f..adafb425babf10ba33a201b4cc57eb6ca9fa1e32 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -7040,6 +7040,31 @@ static SDValue combineToConsecutiveLoads(EVT VT, SDValue Op, const SDLoc &DL, IsAfterLegalize); } +static Constant *getConstantVector(MVT VT, ArrayRef Bits, + const APInt &Undefs, LLVMContext &C) { + unsigned ScalarSize = VT.getScalarSizeInBits(); + Type *Ty = EVT(VT.getScalarType()).getTypeForEVT(C); + + auto getConstantScalar = [&](const APInt &Val) -> Constant * { + if (VT.isFloatingPoint()) { + if (ScalarSize == 16) + return ConstantFP::get(C, APFloat(APFloat::IEEEhalf(), Val)); + if (ScalarSize == 32) + return ConstantFP::get(C, APFloat(APFloat::IEEEsingle(), Val)); + assert(ScalarSize == 64 && "Unsupported floating point scalar size"); + return ConstantFP::get(C, APFloat(APFloat::IEEEdouble(), Val)); + } + return Constant::getIntegerValue(Ty, Val); + }; + + SmallVector ConstantVec; + for (unsigned I = 0, E = Bits.size(); I != E; ++I) + ConstantVec.push_back(Undefs[I] ? UndefValue::get(Ty) + : getConstantScalar(Bits[I])); + + return ConstantVector::get(ArrayRef(ConstantVec)); +} + static Constant *getConstantVector(MVT VT, const APInt &SplatValue, unsigned SplatBitSize, LLVMContext &C) { unsigned ScalarSize = VT.getScalarSizeInBits(); @@ -8727,6 +8752,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { // constants. Insertion into a zero vector is handled as a special-case // somewhere below here. if (NumConstants == NumElems - 1 && NumNonZero != 1 && + FrozenUndefMask.isZero() && (isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT) || isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT))) { // Create an all-constant vector. The variable element in the old @@ -48706,7 +48732,7 @@ static SDValue canonicalizeBitSelect(SDNode *N, SelectionDAG &DAG, if (useVPTERNLOG(Subtarget, VT)) { // Emit a VPTERNLOG node directly - 0xCA is the imm code for A?B:C. // VPTERNLOG is only available as vXi32/64-bit types. - MVT OpSVT = EltSizeInBits == 32 ? MVT::i32 : MVT::i64; + MVT OpSVT = EltSizeInBits <= 32 ? MVT::i32 : MVT::i64; MVT OpVT = MVT::getVectorVT(OpSVT, VT.getSizeInBits() / OpSVT.getSizeInBits()); SDValue A = DAG.getBitcast(OpVT, N0.getOperand(1)); @@ -49885,8 +49911,8 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG, User->getValueSizeInBits(0).getFixedValue() > RegVT.getFixedSizeInBits()) { if (User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD && - cast(User)->getBasePtr() == Ptr && - cast(User)->getMemoryVT().getSizeInBits() == + cast(User)->getBasePtr() == Ptr && + cast(User)->getMemoryVT().getSizeInBits() == MemVT.getSizeInBits()) { SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, SDLoc(N), RegVT.getSizeInBits()); @@ -49914,7 +49940,7 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG, if (ISD::isNormalLoad(User)) { // See if we are loading a constant that matches in the lower // bits of a longer constant (but from a different constant pool ptr). - SDValue UserPtr = cast(User)->getBasePtr(); + SDValue UserPtr = cast(User)->getBasePtr(); const Constant *LdC = getTargetConstantFromBasePtr(Ptr); const Constant *UserC = getTargetConstantFromBasePtr(UserPtr); if (LdC && UserC && UserPtr != Ptr && @@ -54481,6 +54507,8 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, SDValue Op0 = Ops[0]; bool IsSplat = llvm::all_equal(Ops); unsigned NumOps = Ops.size(); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + LLVMContext &Ctx = *DAG.getContext(); // Repeated subvectors. if (IsSplat && @@ -54489,25 +54517,6 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, if (Op0.getOpcode() == X86ISD::VBROADCAST) return DAG.getNode(Op0.getOpcode(), DL, VT, Op0.getOperand(0)); - // If this simple subvector or scalar/subvector broadcast_load is inserted - // into both halves, use a larger broadcast_load. Update other uses to use - // an extracted subvector. - if (ISD::isNormalLoad(Op0.getNode()) || - Op0.getOpcode() == X86ISD::VBROADCAST_LOAD || - Op0.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) { - auto *Mem = cast(Op0); - unsigned Opc = Op0.getOpcode() == X86ISD::VBROADCAST_LOAD - ? X86ISD::VBROADCAST_LOAD - : X86ISD::SUBV_BROADCAST_LOAD; - if (SDValue BcastLd = - getBROADCAST_LOAD(Opc, DL, VT, Mem->getMemoryVT(), Mem, 0, DAG)) { - SDValue BcastSrc = - extractSubVector(BcastLd, 0, DAG, DL, Op0.getValueSizeInBits()); - DAG.ReplaceAllUsesOfValueWith(Op0, BcastSrc); - return BcastLd; - } - } - // concat_vectors(movddup(x),movddup(x)) -> broadcast(x) if (Op0.getOpcode() == X86ISD::MOVDDUP && VT == MVT::v4f64 && (Subtarget.hasAVX2() || @@ -54797,7 +54806,7 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, Subtarget.useAVX512Regs() && Subtarget.getPreferVectorWidth() >= 512 && (SrcVT.getScalarSizeInBits() > 16 || Subtarget.useBWIRegs())) { - EVT NewSrcVT = SrcVT.getDoubleNumVectorElementsVT(*DAG.getContext()); + EVT NewSrcVT = SrcVT.getDoubleNumVectorElementsVT(Ctx); return DAG.getNode(ISD::TRUNCATE, DL, VT, ConcatSubOperand(NewSrcVT, Ops, 0)); } @@ -54954,9 +54963,9 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, (EltSizeInBits >= 32 || Subtarget.hasBWI())) { EVT SelVT = Ops[0].getOperand(0).getValueType(); if (SelVT.getVectorElementType() == MVT::i1) { - SelVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, + SelVT = EVT::getVectorVT(Ctx, MVT::i1, NumOps * SelVT.getVectorNumElements()); - if (DAG.getTargetLoweringInfo().isTypeLegal(SelVT)) + if (TLI.isTypeLegal(SelVT)) return DAG.getNode(Op0.getOpcode(), DL, VT, ConcatSubOperand(SelVT.getSimpleVT(), Ops, 0), ConcatSubOperand(VT, Ops, 1), @@ -54969,8 +54978,8 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, (EltSizeInBits >= 32 || Subtarget.hasInt256()) && IsConcatFree(VT, Ops, 1) && IsConcatFree(VT, Ops, 2)) { EVT SelVT = Ops[0].getOperand(0).getValueType(); - SelVT = SelVT.getDoubleNumVectorElementsVT(*DAG.getContext()); - if (DAG.getTargetLoweringInfo().isTypeLegal(SelVT)) + SelVT = SelVT.getDoubleNumVectorElementsVT(Ctx); + if (TLI.isTypeLegal(SelVT)) return DAG.getNode(Op0.getOpcode(), DL, VT, ConcatSubOperand(SelVT.getSimpleVT(), Ops, 0), ConcatSubOperand(VT, Ops, 1), @@ -54985,7 +54994,7 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, if (auto *FirstLd = dyn_cast(peekThroughBitcasts(Op0))) { unsigned Fast; const X86TargetLowering *TLI = Subtarget.getTargetLowering(); - if (TLI->allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, + if (TLI->allowsMemoryAccess(Ctx, DAG.getDataLayout(), VT, *FirstLd->getMemOperand(), &Fast) && Fast) { if (SDValue Ld = @@ -55002,13 +55011,44 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, APInt OpUndefElts; SmallVector OpEltBits; if (!getTargetConstantBitsFromNode(Ops[I], EltSizeInBits, OpUndefElts, - OpEltBits, true, false)) - break; + OpEltBits, true, false)) + break; EltBits.append(OpEltBits); UndefElts.insertBits(OpUndefElts, I * OpUndefElts.getBitWidth()); } - if (EltBits.size() == VT.getVectorNumElements()) - return getConstVector(EltBits, UndefElts, VT, DAG, DL); + if (EltBits.size() == VT.getVectorNumElements()) { + Constant *C = getConstantVector(VT, EltBits, UndefElts, Ctx); + MVT PVT = TLI.getPointerTy(DAG.getDataLayout()); + SDValue CV = DAG.getConstantPool(C, PVT); + MachineFunction &MF = DAG.getMachineFunction(); + MachinePointerInfo MPI = MachinePointerInfo::getConstantPool(MF); + SDValue Ld = DAG.getLoad(VT, DL, DAG.getEntryNode(), CV, MPI); + SDValue Sub = extractSubVector(Ld, 0, DAG, DL, Op0.getValueSizeInBits()); + DAG.ReplaceAllUsesOfValueWith(Op0, Sub); + return Ld; + } + } + + // If this simple subvector or scalar/subvector broadcast_load is inserted + // into both halves, use a larger broadcast_load. Update other uses to use + // an extracted subvector. + if (IsSplat && + (VT.is256BitVector() || (VT.is512BitVector() && Subtarget.hasAVX512()))) { + if (ISD::isNormalLoad(Op0.getNode()) || + Op0.getOpcode() == X86ISD::VBROADCAST_LOAD || + Op0.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) { + auto *Mem = cast(Op0); + unsigned Opc = Op0.getOpcode() == X86ISD::VBROADCAST_LOAD + ? X86ISD::VBROADCAST_LOAD + : X86ISD::SUBV_BROADCAST_LOAD; + if (SDValue BcastLd = + getBROADCAST_LOAD(Opc, DL, VT, Mem->getMemoryVT(), Mem, 0, DAG)) { + SDValue BcastSrc = + extractSubVector(BcastLd, 0, DAG, DL, Op0.getValueSizeInBits()); + DAG.ReplaceAllUsesOfValueWith(Op0, BcastSrc); + return BcastLd; + } + } } // If we're splatting a 128-bit subvector to 512-bits, use SHUF128 directly. diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp index a69ce40982ad826b42a2615c5642a6c5c583f9ee..d6ae4971d238d836f4f6d17f1fbcbc7d30551cb5 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -8717,11 +8717,6 @@ bool X86InstrInfo::isSafeToMoveRegClassDefs( /// TODO: Eliminate this and move the code to X86MachineFunctionInfo. /// unsigned X86InstrInfo::getGlobalBaseReg(MachineFunction *MF) const { - assert((!Subtarget.is64Bit() || - MF->getTarget().getCodeModel() == CodeModel::Medium || - MF->getTarget().getCodeModel() == CodeModel::Large) && - "X86-64 PIC uses RIP relative addressing"); - X86MachineFunctionInfo *X86FI = MF->getInfo(); Register GlobalBaseReg = X86FI->getGlobalBaseReg(); if (GlobalBaseReg != 0) @@ -10083,12 +10078,6 @@ struct CGBR : public MachineFunctionPass { static_cast(&MF.getTarget()); const X86Subtarget &STI = MF.getSubtarget(); - // Don't do anything in the 64-bit small and kernel code models. They use - // RIP-relative addressing for everything. - if (STI.is64Bit() && (TM->getCodeModel() == CodeModel::Small || - TM->getCodeModel() == CodeModel::Kernel)) - return false; - // Only emit a global base reg in PIC mode. if (!TM->isPositionIndependent()) return false; @@ -10114,16 +10103,7 @@ struct CGBR : public MachineFunctionPass { PC = GlobalBaseReg; if (STI.is64Bit()) { - if (TM->getCodeModel() == CodeModel::Medium) { - // In the medium code model, use a RIP-relative LEA to materialize the - // GOT. - BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PC) - .addReg(X86::RIP) - .addImm(0) - .addReg(0) - .addExternalSymbol("_GLOBAL_OFFSET_TABLE_") - .addReg(0); - } else if (TM->getCodeModel() == CodeModel::Large) { + if (TM->getCodeModel() == CodeModel::Large) { // In the large code model, we are aiming for this code, though the // register allocation may vary: // leaq .LN$pb(%rip), %rax @@ -10146,7 +10126,14 @@ struct CGBR : public MachineFunctionPass { .addReg(PBReg, RegState::Kill) .addReg(GOTReg, RegState::Kill); } else { - llvm_unreachable("unexpected code model"); + // In other code models, use a RIP-relative LEA to materialize the + // GOT. + BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PC) + .addReg(X86::RIP) + .addImm(0) + .addReg(0) + .addExternalSymbol("_GLOBAL_OFFSET_TABLE_") + .addReg(0); } } else { // Operand of MovePCtoStack is completely ignored by asm printer. It's diff --git a/llvm/lib/Target/X86/X86InstrSSE.td b/llvm/lib/Target/X86/X86InstrSSE.td index 34eb17af1033d72eee36fd152b3153646e548b92..cf57fe562ed5c458a37ad96468973a6e2f93a254 100644 --- a/llvm/lib/Target/X86/X86InstrSSE.td +++ b/llvm/lib/Target/X86/X86InstrSSE.td @@ -7093,35 +7093,35 @@ def VBROADCASTSDYrr : avx2_broadcast_rr<0x19, "vbroadcastsd", VR256, // halves of a 256-bit vector. // let mayLoad = 1, hasSideEffects = 0, Predicates = [HasAVX2] in -def VBROADCASTI128 : AVX8I<0x5A, MRMSrcMem, (outs VR256:$dst), - (ins i128mem:$src), - "vbroadcasti128\t{$src, $dst|$dst, $src}", []>, - Sched<[WriteShuffleLd]>, VEX, VEX_L; +def VBROADCASTI128rm : AVX8I<0x5A, MRMSrcMem, (outs VR256:$dst), + (ins i128mem:$src), + "vbroadcasti128\t{$src, $dst|$dst, $src}", []>, + Sched<[WriteShuffleLd]>, VEX, VEX_L; let mayLoad = 1, hasSideEffects = 0, Predicates = [HasAVX], ExeDomain = SSEPackedSingle in -def VBROADCASTF128 : AVX8I<0x1A, MRMSrcMem, (outs VR256:$dst), - (ins f128mem:$src), - "vbroadcastf128\t{$src, $dst|$dst, $src}", []>, - Sched<[SchedWriteFShuffle.XMM.Folded]>, VEX, VEX_L; +def VBROADCASTF128rm : AVX8I<0x1A, MRMSrcMem, (outs VR256:$dst), + (ins f128mem:$src), + "vbroadcastf128\t{$src, $dst|$dst, $src}", []>, + Sched<[SchedWriteFShuffle.XMM.Folded]>, VEX, VEX_L; let Predicates = [HasAVX, NoVLX] in { def : Pat<(v4f64 (X86SubVBroadcastld128 addr:$src)), - (VBROADCASTF128 addr:$src)>; + (VBROADCASTF128rm addr:$src)>; def : Pat<(v8f32 (X86SubVBroadcastld128 addr:$src)), - (VBROADCASTF128 addr:$src)>; + (VBROADCASTF128rm addr:$src)>; // NOTE: We're using FP instructions here, but execution domain fixing can // convert to integer when profitable. def : Pat<(v4i64 (X86SubVBroadcastld128 addr:$src)), - (VBROADCASTF128 addr:$src)>; + (VBROADCASTF128rm addr:$src)>; def : Pat<(v8i32 (X86SubVBroadcastld128 addr:$src)), - (VBROADCASTF128 addr:$src)>; + (VBROADCASTF128rm addr:$src)>; def : Pat<(v16i16 (X86SubVBroadcastld128 addr:$src)), - (VBROADCASTF128 addr:$src)>; + (VBROADCASTF128rm addr:$src)>; def : Pat<(v16f16 (X86SubVBroadcastld128 addr:$src)), - (VBROADCASTF128 addr:$src)>; + (VBROADCASTF128rm addr:$src)>; def : Pat<(v32i8 (X86SubVBroadcastld128 addr:$src)), - (VBROADCASTF128 addr:$src)>; + (VBROADCASTF128rm addr:$src)>; } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/X86/X86InstrSystem.td b/llvm/lib/Target/X86/X86InstrSystem.td index b55956169ff2cfed9086e1253ad8c17b36324f2e..cbb5d4ed5bbdcde164c2f00726354fa60d724245 100644 --- a/llvm/lib/Target/X86/X86InstrSystem.td +++ b/llvm/lib/Target/X86/X86InstrSystem.td @@ -679,6 +679,10 @@ def INVPCID32 : I<0x82, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2), def INVPCID64 : I<0x82, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invpcid\t{$src2, $src1|$src1, $src2}", []>, T8PD, Requires<[In64BitMode, HasINVPCID]>; + +def INVPCID64_EVEX : I<0xF2, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), + "invpcid\t{$src2, $src1|$src1, $src2}", []>, + EVEX_NoCD8, T_MAP4XS, Requires<[In64BitMode, HasINVPCID]>; } // SchedRW let Predicates = [In64BitMode, HasINVPCID] in { diff --git a/llvm/lib/Target/X86/X86InstrUtils.td b/llvm/lib/Target/X86/X86InstrUtils.td index 071c41da672200a3237b8e371a3920fc56d4625d..2f056f2ead62fe94667eabb7aaf0809101bbb9cb 100644 --- a/llvm/lib/Target/X86/X86InstrUtils.td +++ b/llvm/lib/Target/X86/X86InstrUtils.td @@ -86,6 +86,7 @@ class EVEX_CD8 { int CD8_EltSize = !srl(esize, 3); bits<3> CD8_Form = form.Value; } +class EVEX_NoCD8 : EVEX { bits<7> CD8_Scale = 0; } class XOP { Encoding OpEnc = EncXOP; } class XOP_4V : XOP { bit hasVEX_4V = 1; } class EVEX2VEXOverride { diff --git a/llvm/lib/Target/X86/X86InstrVMX.td b/llvm/lib/Target/X86/X86InstrVMX.td index c601b1a97932b1ab4c35b510e31847093c84c5a4..5289819119ceb3fe215e8fb665442d76106748a1 100644 --- a/llvm/lib/Target/X86/X86InstrVMX.td +++ b/llvm/lib/Target/X86/X86InstrVMX.td @@ -22,11 +22,9 @@ def INVEPT32 : I<0x80, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2), def INVEPT64 : I<0x80, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invept\t{$src2, $src1|$src1, $src2}", []>, T8PD, Requires<[In64BitMode]>; - -let CD8_Scale = 0 in def INVEPT64_EVEX : I<0xF0, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invept\t{$src2, $src1|$src1, $src2}", []>, - EVEX, T_MAP4XS, Requires<[In64BitMode]>; + EVEX_NoCD8, T_MAP4XS, Requires<[In64BitMode]>; // 66 0F 38 81 def INVVPID32 : I<0x81, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2), @@ -35,11 +33,9 @@ def INVVPID32 : I<0x81, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2), def INVVPID64 : I<0x81, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invvpid\t{$src2, $src1|$src1, $src2}", []>, T8PD, Requires<[In64BitMode]>; - -let CD8_Scale = 0 in def INVVPID64_EVEX : I<0xF1, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invvpid\t{$src2, $src1|$src1, $src2}", []>, - EVEX, T_MAP4XS, Requires<[In64BitMode]>; + EVEX_NoCD8, T_MAP4XS, Requires<[In64BitMode]>; // 0F 01 C1 def VMCALL : I<0x01, MRM_C1, (outs), (ins), "vmcall", []>, TB; diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp index cbb0121615241696dbf7a4dfb0281e1d1b5d618c..e1a67f61e7664092262a6a02830f7a647e8705c7 100644 --- a/llvm/lib/Target/X86/X86MCInstLower.cpp +++ b/llvm/lib/Target/X86/X86MCInstLower.cpp @@ -1865,8 +1865,8 @@ static void addConstantComments(const MachineInstr *MI, // For loads from a constant pool to a vector register, print the constant // loaded. CASE_ALL_MOV_RM() - case X86::VBROADCASTF128: - case X86::VBROADCASTI128: + case X86::VBROADCASTF128rm: + case X86::VBROADCASTI128rm: case X86::VBROADCASTF32X4Z256rm: case X86::VBROADCASTF32X4rm: case X86::VBROADCASTF32X8rm: @@ -1891,8 +1891,8 @@ static void addConstantComments(const MachineInstr *MI, CASE_128_MOV_RM() NumLanes = 1; BitWidth = 128; break; CASE_256_MOV_RM() NumLanes = 1; BitWidth = 256; break; CASE_512_MOV_RM() NumLanes = 1; BitWidth = 512; break; - case X86::VBROADCASTF128: NumLanes = 2; BitWidth = 128; break; - case X86::VBROADCASTI128: NumLanes = 2; BitWidth = 128; break; + case X86::VBROADCASTF128rm: NumLanes = 2; BitWidth = 128; break; + case X86::VBROADCASTI128rm: NumLanes = 2; BitWidth = 128; break; case X86::VBROADCASTF32X4Z256rm: NumLanes = 2; BitWidth = 128; break; case X86::VBROADCASTF32X4rm: NumLanes = 4; BitWidth = 128; break; case X86::VBROADCASTF32X8rm: NumLanes = 2; BitWidth = 256; break; diff --git a/llvm/lib/Target/X86/X86ReplaceableInstrs.def b/llvm/lib/Target/X86/X86ReplaceableInstrs.def index 4798275c051923a6afa8292e274993fb7715ad7b..e1383198d3fe96bb535c60833dbbca3c9740912c 100644 --- a/llvm/lib/Target/X86/X86ReplaceableInstrs.def +++ b/llvm/lib/Target/X86/X86ReplaceableInstrs.def @@ -202,7 +202,7 @@ ENTRY(VBROADCASTSSYrr, VBROADCASTSSYrr, VPBROADCASTDYrr) ENTRY(VBROADCASTSSYrm, VBROADCASTSSYrm, VPBROADCASTDYrm) ENTRY(VBROADCASTSDYrr, VBROADCASTSDYrr, VPBROADCASTQYrr) ENTRY(VBROADCASTSDYrm, VBROADCASTSDYrm, VPBROADCASTQYrm) -ENTRY(VBROADCASTF128, VBROADCASTF128, VBROADCASTI128) +ENTRY(VBROADCASTF128rm, VBROADCASTF128rm, VBROADCASTI128rm) ENTRY(VBLENDPSYrri, VBLENDPSYrri, VPBLENDDYrri) ENTRY(VBLENDPSYrmi, VBLENDPSYrmi, VPBLENDDYrmi) ENTRY(VPERMILPSYmi, VPERMILPSYmi, VPSHUFDYmi) diff --git a/llvm/lib/Target/X86/X86SchedAlderlakeP.td b/llvm/lib/Target/X86/X86SchedAlderlakeP.td index 3406a28be2c29a2a1f7bff7718d7e94d3bbdb721..8e3e55428264804dcad62a93b052910c5caf92ca 100644 --- a/llvm/lib/Target/X86/X86SchedAlderlakeP.td +++ b/llvm/lib/Target/X86/X86SchedAlderlakeP.td @@ -1328,7 +1328,7 @@ def ADLPWriteResGroup117 : SchedWriteRes<[ADLPPort02_03_11]> { let Latency = 8; } def : InstRW<[ADLPWriteResGroup117], (instregex "^MMX_MOV(D|Q)64rm$", - "^VBROADCAST(F|I)128$", + "^VBROADCAST(F|I)128rm$", "^VBROADCASTS(D|S)Yrm$", "^VMOV(D|SH|SL)DUPYrm$", "^VPBROADCAST(D|Q)Yrm$")>; diff --git a/llvm/lib/Target/X86/X86SchedBroadwell.td b/llvm/lib/Target/X86/X86SchedBroadwell.td index 8575f747b4d3bc7873347bd318910e2b68c3540e..61a8832000e2fadedb8084fed1f10cd3af76e3b7 100644 --- a/llvm/lib/Target/X86/X86SchedBroadwell.td +++ b/llvm/lib/Target/X86/X86SchedBroadwell.td @@ -946,8 +946,8 @@ def BWWriteResGroup58 : SchedWriteRes<[BWPort23]> { let ReleaseAtCycles = [1]; } def: InstRW<[BWWriteResGroup58], (instregex "LD_F(32|64|80)m")>; -def: InstRW<[BWWriteResGroup58], (instrs VBROADCASTF128, - VBROADCASTI128, +def: InstRW<[BWWriteResGroup58], (instrs VBROADCASTF128rm, + VBROADCASTI128rm, VBROADCASTSDYrm, VBROADCASTSSYrm, VMOVDDUPYrm, diff --git a/llvm/lib/Target/X86/X86SchedHaswell.td b/llvm/lib/Target/X86/X86SchedHaswell.td index d10d7684ac12730fe9c0e2725d6b63c98c4e1a62..8795ca95c5593766bbd962ee9255e162b825d57f 100644 --- a/llvm/lib/Target/X86/X86SchedHaswell.td +++ b/llvm/lib/Target/X86/X86SchedHaswell.td @@ -876,8 +876,8 @@ def HWWriteResGroup0_1 : SchedWriteRes<[HWPort23]> { let NumMicroOps = 1; let ReleaseAtCycles = [1]; } -def: InstRW<[HWWriteResGroup0_1], (instrs VBROADCASTF128, - VBROADCASTI128, +def: InstRW<[HWWriteResGroup0_1], (instrs VBROADCASTF128rm, + VBROADCASTI128rm, VBROADCASTSDYrm, VBROADCASTSSYrm, VMOVDDUPYrm, diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td index a2aa2655bca2856112709d42d787e7e2fc2a0149..e27af1433d4559fb302fdc9a0bff0c15e60e273a 100644 --- a/llvm/lib/Target/X86/X86SchedIceLake.td +++ b/llvm/lib/Target/X86/X86SchedIceLake.td @@ -1274,8 +1274,8 @@ def ICXWriteResGroup89 : SchedWriteRes<[ICXPort23]> { let ReleaseAtCycles = [1]; } def: InstRW<[ICXWriteResGroup89], (instregex "LD_F(32|64|80)m")>; -def: InstRW<[ICXWriteResGroup89], (instrs VBROADCASTF128, - VBROADCASTI128, +def: InstRW<[ICXWriteResGroup89], (instrs VBROADCASTF128rm, + VBROADCASTI128rm, VBROADCASTSDYrm, VBROADCASTSSYrm, VMOVDDUPYrm, diff --git a/llvm/lib/Target/X86/X86SchedSapphireRapids.td b/llvm/lib/Target/X86/X86SchedSapphireRapids.td index 6a426ef4cf54330892bd83f03342014042afa88c..4eac53385ae54deeb6d587af2790951426e7d628 100644 --- a/llvm/lib/Target/X86/X86SchedSapphireRapids.td +++ b/llvm/lib/Target/X86/X86SchedSapphireRapids.td @@ -1599,7 +1599,7 @@ def SPRWriteResGroup126 : SchedWriteRes<[SPRPort02_03_11]> { let Latency = 8; } def : InstRW<[SPRWriteResGroup126], (instregex "^MMX_MOV(D|Q)64rm$", - "^VBROADCAST(F|I)128$", + "^VBROADCAST(F|I)128rm$", "^VBROADCAST(F|I)32X(2|4)Z256rm$", "^VBROADCAST(F|I)32X(8|2Z)rm$", "^VBROADCAST(F|I)(32|64)X4rm$", diff --git a/llvm/lib/Target/X86/X86SchedSkylakeClient.td b/llvm/lib/Target/X86/X86SchedSkylakeClient.td index 92ed491bc296133459347bb45df8a6c891f9d766..4fa138f69fb92b3bd7b61f0c36b145e1d29c709d 100644 --- a/llvm/lib/Target/X86/X86SchedSkylakeClient.td +++ b/llvm/lib/Target/X86/X86SchedSkylakeClient.td @@ -1064,8 +1064,8 @@ def SKLWriteResGroup85 : SchedWriteRes<[SKLPort23]> { let ReleaseAtCycles = [1]; } def: InstRW<[SKLWriteResGroup85], (instregex "LD_F(32|64|80)m")>; -def: InstRW<[SKLWriteResGroup85], (instrs VBROADCASTF128, - VBROADCASTI128, +def: InstRW<[SKLWriteResGroup85], (instrs VBROADCASTF128rm, + VBROADCASTI128rm, VBROADCASTSDYrm, VBROADCASTSSYrm, VMOVDDUPYrm, diff --git a/llvm/lib/Target/X86/X86SchedSkylakeServer.td b/llvm/lib/Target/X86/X86SchedSkylakeServer.td index ed22f95c83e58201a95596e4e5f729d05a2e9e88..8194af8a6e1db2bb202f709d98d60b34276b9436 100644 --- a/llvm/lib/Target/X86/X86SchedSkylakeServer.td +++ b/llvm/lib/Target/X86/X86SchedSkylakeServer.td @@ -1254,8 +1254,8 @@ def SKXWriteResGroup89 : SchedWriteRes<[SKXPort23]> { let ReleaseAtCycles = [1]; } def: InstRW<[SKXWriteResGroup89], (instregex "LD_F(32|64|80)m")>; -def: InstRW<[SKXWriteResGroup89], (instrs VBROADCASTF128, - VBROADCASTI128, +def: InstRW<[SKXWriteResGroup89], (instrs VBROADCASTF128rm, + VBROADCASTI128rm, VBROADCASTSDYrm, VBROADCASTSSYrm, VMOVDDUPYrm, diff --git a/llvm/lib/Target/X86/X86ScheduleBdVer2.td b/llvm/lib/Target/X86/X86ScheduleBdVer2.td index aeeabba45b65ec8e1b9fbe2fa3055de398173db9..c9749979576f2728d7e892c94ea7d76eaf2e383f 100644 --- a/llvm/lib/Target/X86/X86ScheduleBdVer2.td +++ b/llvm/lib/Target/X86/X86ScheduleBdVer2.td @@ -933,7 +933,7 @@ def PdWriteVBROADCASTF128 : SchedWriteRes<[PdFPU01, PdFPFMA]> { let ReleaseAtCycles = [1, 3]; let NumMicroOps = 2; } -def : InstRW<[PdWriteVBROADCASTF128], (instrs VBROADCASTF128)>; +def : InstRW<[PdWriteVBROADCASTF128], (instrs VBROADCASTF128rm)>; defm : PdWriteResXMMPair; defm : PdWriteResYMMPair; diff --git a/llvm/lib/Target/X86/X86ScheduleBtVer2.td b/llvm/lib/Target/X86/X86ScheduleBtVer2.td index 8b7d501981b1e63ea2c37ae3c3fd3701a3561c0d..9cba933e82b0547bd7590cc88112a09383551929 100644 --- a/llvm/lib/Target/X86/X86ScheduleBtVer2.td +++ b/llvm/lib/Target/X86/X86ScheduleBtVer2.td @@ -816,7 +816,7 @@ def JWriteVBROADCASTYLd: SchedWriteRes<[JLAGU, JFPU01, JFPX]> { } def : InstRW<[JWriteVBROADCASTYLd], (instrs VBROADCASTSDYrm, VBROADCASTSSYrm, - VBROADCASTF128)>; + VBROADCASTF128rm)>; def JWriteJVZEROALL: SchedWriteRes<[]> { let Latency = 90; diff --git a/llvm/lib/Target/X86/X86ScheduleZnver1.td b/llvm/lib/Target/X86/X86ScheduleZnver1.td index e39bcf807a0c0022a124b5d5a70d56b3371cf7d2..7ee9eadf8439085ff6b9a95841bcbd8c1f6d9fe8 100644 --- a/llvm/lib/Target/X86/X86ScheduleZnver1.td +++ b/llvm/lib/Target/X86/X86ScheduleZnver1.td @@ -996,8 +996,8 @@ def ZnWriteBROADCAST : SchedWriteRes<[ZnAGU, ZnFPU13]> { let Latency = 8; } // VBROADCASTF128 / VBROADCASTI128. -def : InstRW<[ZnWriteBROADCAST], (instrs VBROADCASTF128, - VBROADCASTI128)>; +def : InstRW<[ZnWriteBROADCAST], (instrs VBROADCASTF128rm, + VBROADCASTI128rm)>; // EXTRACTPS. // r32,x,i. diff --git a/llvm/lib/Target/X86/X86ScheduleZnver2.td b/llvm/lib/Target/X86/X86ScheduleZnver2.td index ecaca70bb67755cb82c5cbbb5f33215e1cb92a25..c0775847798d27d05104e40ee4901abcd969f146 100644 --- a/llvm/lib/Target/X86/X86ScheduleZnver2.td +++ b/llvm/lib/Target/X86/X86ScheduleZnver2.td @@ -1004,8 +1004,8 @@ def Zn2WriteBROADCAST : SchedWriteRes<[Zn2AGU, Zn2FPU13]> { let Latency = 8; } // VBROADCASTF128 / VBROADCASTI128. -def : InstRW<[Zn2WriteBROADCAST], (instrs VBROADCASTF128, - VBROADCASTI128)>; +def : InstRW<[Zn2WriteBROADCAST], (instrs VBROADCASTF128rm, + VBROADCASTI128rm)>; // EXTRACTPS. // r32,x,i. diff --git a/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp b/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp index 2a896314875d29376c8400e37e7e44c80a4156fc..6301285fe954575372b907579edfd1c78edbaabe 100644 --- a/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp +++ b/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp @@ -26,7 +26,6 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp index ae47929fce6aceb6a924f5d2c40377e208501e85..40f4ba0be5503a88b95e06fd48338b97ec1a1614 100644 --- a/llvm/lib/TargetParser/Host.cpp +++ b/llvm/lib/TargetParser/Host.cpp @@ -202,6 +202,7 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) { .Case("0xc20", "cortex-m0") .Case("0xc23", "cortex-m3") .Case("0xc24", "cortex-m4") + .Case("0xd24", "cortex-m52") .Case("0xd22", "cortex-m55") .Case("0xd02", "cortex-a34") .Case("0xd04", "cortex-a35") diff --git a/llvm/lib/Transforms/Hello/CMakeLists.txt b/llvm/lib/Transforms/Hello/CMakeLists.txt index c4f10247c1a6ccab7d9d872761702c834282fcc8..9510c31f633fe8bc6bc0dc7d393d187a542d432c 100644 --- a/llvm/lib/Transforms/Hello/CMakeLists.txt +++ b/llvm/lib/Transforms/Hello/CMakeLists.txt @@ -6,7 +6,7 @@ if( NOT LLVM_REQUIRES_RTTI ) endif() endif() -if(WIN32 OR CYGWIN) +if(WIN32 OR CYGWIN OR ZOS) set(LLVM_LINK_COMPONENTS Core Support) endif() diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index 889ebd7438bd59c6628e7aa16d9db530eac19078..cbe0a96976c3f4465dcc341ed7d5a6055016a930 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -290,20 +290,19 @@ static const Value *getPointerOperand(const Instruction *I, return nullptr; } -/// Helper function to create a pointer of type \p ResTy, based on \p Ptr, and -/// advanced by \p Offset bytes. To aid later analysis the method tries to build +/// Helper function to create a pointer based on \p Ptr, and advanced by \p +/// Offset bytes. To aid later analysis the method tries to build /// getelement pointer instructions that traverse the natural type of \p Ptr if /// possible. If that fails, the remaining offset is adjusted byte-wise, hence /// through a cast to i8*. /// /// TODO: This could probably live somewhere more prominantly if it doesn't /// already exist. -static Value *constructPointer(Type *ResTy, Type *PtrElemTy, Value *Ptr, - int64_t Offset, IRBuilder &IRB, - const DataLayout &DL) { +static Value *constructPointer(Type *PtrElemTy, Value *Ptr, int64_t Offset, + IRBuilder &IRB, const DataLayout &DL) { assert(Offset >= 0 && "Negative offset not supported yet!"); LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset - << "-bytes as " << *ResTy << "\n"); + << "-bytes\n"); if (Offset) { Type *Ty = PtrElemTy; @@ -327,10 +326,6 @@ static Value *constructPointer(Type *ResTy, Type *PtrElemTy, Value *Ptr, } } - // Ensure the result has the requested type. - Ptr = IRB.CreatePointerBitCastOrAddrSpaceCast(Ptr, ResTy, - Ptr->getName() + ".cast"); - LLVM_DEBUG(dbgs() << "Constructed pointer: " << *Ptr << "\n"); return Ptr; } @@ -7492,19 +7487,16 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { if (auto *PrivStructType = dyn_cast(PrivType)) { const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType); for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) { - Type *PointeeTy = PrivStructType->getElementType(u)->getPointerTo(); - Value *Ptr = - constructPointer(PointeeTy, PrivType, &Base, - PrivStructLayout->getElementOffset(u), IRB, DL); + Value *Ptr = constructPointer( + PrivType, &Base, PrivStructLayout->getElementOffset(u), IRB, DL); new StoreInst(F.getArg(ArgNo + u), Ptr, &IP); } } else if (auto *PrivArrayType = dyn_cast(PrivType)) { Type *PointeeTy = PrivArrayType->getElementType(); - Type *PointeePtrTy = PointeeTy->getPointerTo(); uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy); for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) { - Value *Ptr = constructPointer(PointeePtrTy, PrivType, &Base, - u * PointeeTySize, IRB, DL); + Value *Ptr = + constructPointer(PrivType, &Base, u * PointeeTySize, IRB, DL); new StoreInst(F.getArg(ArgNo + u), Ptr, &IP); } } else { @@ -7524,19 +7516,13 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { IRBuilder IRB(IP); const DataLayout &DL = IP->getModule()->getDataLayout(); - Type *PrivPtrType = PrivType->getPointerTo(); - if (Base->getType() != PrivPtrType) - Base = BitCastInst::CreatePointerBitCastOrAddrSpaceCast( - Base, PrivPtrType, "", ACS.getInstruction()); - // Traverse the type, build GEPs and loads. if (auto *PrivStructType = dyn_cast(PrivType)) { const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType); for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) { Type *PointeeTy = PrivStructType->getElementType(u); - Value *Ptr = - constructPointer(PointeeTy->getPointerTo(), PrivType, Base, - PrivStructLayout->getElementOffset(u), IRB, DL); + Value *Ptr = constructPointer( + PrivType, Base, PrivStructLayout->getElementOffset(u), IRB, DL); LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP); L->setAlignment(Alignment); ReplacementValues.push_back(L); @@ -7544,10 +7530,9 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { } else if (auto *PrivArrayType = dyn_cast(PrivType)) { Type *PointeeTy = PrivArrayType->getElementType(); uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy); - Type *PointeePtrTy = PointeeTy->getPointerTo(); for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) { - Value *Ptr = constructPointer(PointeePtrTy, PrivType, Base, - u * PointeeTySize, IRB, DL); + Value *Ptr = + constructPointer(PrivType, Base, u * PointeeTySize, IRB, DL); LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP); L->setAlignment(Alignment); ReplacementValues.push_back(L); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp index 39466a9e3751bb1071c3695f2038886eae6c1ec6..6002f599ca71ab32ccbdce3d7d77435f98e3f6dc 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp @@ -2127,10 +2127,10 @@ Instruction *InstCombinerImpl::foldBinOpOfDisplacedShifts(BinaryOperator &I) { Constant *ShiftedC1, *ShiftedC2, *AddC; Type *Ty = I.getType(); unsigned BitWidth = Ty->getScalarSizeInBits(); - if (!match(&I, - m_c_BinOp(m_Shift(m_ImmConstant(ShiftedC1), m_Value(ShAmt)), - m_Shift(m_ImmConstant(ShiftedC2), - m_Add(m_Deferred(ShAmt), m_ImmConstant(AddC)))))) + if (!match(&I, m_c_BinOp(m_Shift(m_ImmConstant(ShiftedC1), m_Value(ShAmt)), + m_Shift(m_ImmConstant(ShiftedC2), + m_AddLike(m_Deferred(ShAmt), + m_ImmConstant(AddC)))))) return nullptr; // Make sure the add constant is a valid shift amount. diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index a991f0906052a30991c78fb37e6f848cb4bb32a0..255ce6973a16fbac643345bd49f9095e7b391bb1 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -514,6 +514,8 @@ static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC) { return IC.replaceInstUsesWith(II, ConstantInt::getNullValue(II.getType())); } + Constant *C; + if (IsTZ) { // cttz(-x) -> cttz(x) if (match(Op0, m_Neg(m_Value(X)))) @@ -549,6 +551,38 @@ static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC) { if (match(Op0, m_Intrinsic(m_Value(X)))) return IC.replaceOperand(II, 0, X); + + // cttz(shl(%const, %val), 1) --> add(cttz(%const, 1), %val) + if (match(Op0, m_Shl(m_ImmConstant(C), m_Value(X))) && + match(Op1, m_One())) { + Value *ConstCttz = + IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, C, Op1); + return BinaryOperator::CreateAdd(ConstCttz, X); + } + + // cttz(lshr exact (%const, %val), 1) --> sub(cttz(%const, 1), %val) + if (match(Op0, m_Exact(m_LShr(m_ImmConstant(C), m_Value(X)))) && + match(Op1, m_One())) { + Value *ConstCttz = + IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, C, Op1); + return BinaryOperator::CreateSub(ConstCttz, X); + } + } else { + // ctlz(lshr(%const, %val), 1) --> add(ctlz(%const, 1), %val) + if (match(Op0, m_LShr(m_ImmConstant(C), m_Value(X))) && + match(Op1, m_One())) { + Value *ConstCtlz = + IC.Builder.CreateBinaryIntrinsic(Intrinsic::ctlz, C, Op1); + return BinaryOperator::CreateAdd(ConstCtlz, X); + } + + // ctlz(shl nuw (%const, %val), 1) --> sub(ctlz(%const, 1), %val) + if (match(Op0, m_NUWShl(m_ImmConstant(C), m_Value(X))) && + match(Op1, m_One())) { + Value *ConstCtlz = + IC.Builder.CreateBinaryIntrinsic(Intrinsic::ctlz, C, Op1); + return BinaryOperator::CreateSub(ConstCtlz, X); + } } KnownBits Known = IC.computeKnownBits(Op0, 0, &II); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp index 4295c7692e6d52b993ca44a8714eb69402960817..91642e3babceec2bbd2652b66c8aa10d4b8d41ae 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -23,7 +23,6 @@ #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/GetElementPtrTypeIterator.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/PatternMatch.h" #include "llvm/Support/KnownBits.h" @@ -812,8 +811,8 @@ Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, // Only lower this if the icmp is the only user of the GEP or if we expect // the result to fold to a constant! if ((GEPsInBounds || CmpInst::isEquality(Cond)) && - (isa(GEPLHS) || GEPLHS->hasOneUse()) && - (isa(GEPRHS) || GEPRHS->hasOneUse())) { + (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) && + (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse())) { // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) ---> (OFFSET1 cmp OFFSET2) Value *L = EmitGEPOffset(GEPLHS); Value *R = EmitGEPOffset(GEPRHS); @@ -7034,7 +7033,7 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) { return Res; { - Value *X, *Y, *Z; + Value *X, *Y; // Transform (X & ~Y) == 0 --> (X & Y) != 0 // and (X & ~Y) != 0 --> (X & Y) == 0 // if A is a power of 2. @@ -7044,32 +7043,18 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) { return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(X, Y), Op1); - // Transform (~X ^ Y) s< ~Z --> (X ^ Y) s> Z, - // (~X ^ Y) s> ~Z --> (X ^ Y) s< Z, - // (~X ^ Y) s<= ~Z --> (X ^ Y) s>= Z, - // (~X ^ Y) s>= ~Z --> (X ^ Y) s<= Z, - // (~X ^ Y) u< ~Z --> (X ^ Y) u< Z, - // (~X ^ Y) u> ~Z --> (X ^ Y) u< Z, - // (~X ^ Y) u<= ~Z --> (X ^ Y) u>= Z, - // (~X ^ Y) u>= ~Z --> (X ^ Y) u<= Z, - // (~X ^ Y) == ~Z --> (X ^ Y) == Z, - // and (~X ^ Y) != ~Z --> (X ^ Y) != Z, - if (match(&I, m_c_ICmp(Pred, m_c_Xor(m_Not(m_Value(X)), m_Value(Y)), - m_Not(m_Value(Z)))) && - (I.getOperand(0)->hasOneUse() || I.getOperand(1)->hasOneUse())) - return new ICmpInst(I.getSwappedPredicate(Pred), Builder.CreateXor(X, Y), - Z); - - // ~X < ~Y --> Y < X - // ~X < C --> X > ~C - if (match(Op0, m_Not(m_Value(X)))) { - if (match(Op1, m_Not(m_Value(Y)))) - return new ICmpInst(I.getPredicate(), Y, X); - - const APInt *C; - if (match(Op1, m_APInt(C))) - return new ICmpInst(I.getSwappedPredicate(), X, - ConstantInt::get(Op1->getType(), ~(*C))); + // Op0 pred Op1 -> ~Op1 pred ~Op0, if this allows us to drop an instruction. + if (Op0->getType()->isIntOrIntVectorTy()) { + bool ConsumesOp0, ConsumesOp1; + if (isFreeToInvert(Op0, Op0->hasOneUse(), ConsumesOp0) && + isFreeToInvert(Op1, Op1->hasOneUse(), ConsumesOp1) && + (ConsumesOp0 || ConsumesOp1)) { + Value *InvOp0 = getFreelyInverted(Op0, Op0->hasOneUse(), &Builder); + Value *InvOp1 = getFreelyInverted(Op1, Op1->hasOneUse(), &Builder); + assert(InvOp0 && InvOp1 && + "Mismatch between isFreeToInvert and getFreelyInverted"); + return new ICmpInst(I.getSwappedPredicate(), InvOp0, InvOp1); + } } Instruction *AddI = nullptr; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp index b72b68c68d985bd498d90bad8c8ba53f58aceb00..bb2a77daa60a764ede5836dddc17d6966282294f 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -213,29 +213,10 @@ static Instruction *simplifyAllocaArraySize(InstCombinerImpl &IC, AllocaInst *New = IC.Builder.CreateAlloca(NewTy, AI.getAddressSpace(), nullptr, AI.getName()); New->setAlignment(AI.getAlign()); + New->setUsedWithInAlloca(AI.isUsedWithInAlloca()); replaceAllDbgUsesWith(AI, *New, *New, DT); - - // Scan to the end of the allocation instructions, to skip over a block of - // allocas if possible...also skip interleaved debug info - // - BasicBlock::iterator It(New); - while (isa(*It) || isa(*It)) - ++It; - - // Now that I is pointing to the first non-allocation-inst in the block, - // insert our getelementptr instruction... - // - Type *IdxTy = IC.getDataLayout().getIndexType(AI.getType()); - Value *NullIdx = Constant::getNullValue(IdxTy); - Value *Idx[2] = {NullIdx, NullIdx}; - Instruction *GEP = GetElementPtrInst::CreateInBounds( - NewTy, New, Idx, New->getName() + ".sub"); - IC.InsertNewInstBefore(GEP, It); - - // Now make everything use the getelementptr instead of the original - // allocation. - return IC.replaceInstUsesWith(AI, GEP); + return IC.replaceInstUsesWith(AI, New); } } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp index 588bb00462d270a99586d821443886b0c185b046..8d5866e98a8ef9c7f15f87abd42fa83e03cb9d99 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -300,8 +300,7 @@ Instruction *InstCombinerImpl::visitMul(BinaryOperator &I) { // Canonicalize (X|C1)*MulC -> X*MulC+C1*MulC. Value *X; Constant *C1; - if (match(Op0, m_OneUse(m_Add(m_Value(X), m_ImmConstant(C1)))) || - match(Op0, m_OneUse(m_DisjointOr(m_Value(X), m_ImmConstant(C1))))) { + if (match(Op0, m_OneUse(m_AddLike(m_Value(X), m_ImmConstant(C1))))) { // C1*MulC simplifies to a tidier constant. Value *NewC = Builder.CreateMul(C1, MulC); auto *BOp0 = cast(Op0); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp index 513b185c83a49e9defcac69adc43cec3b7e707b1..62e49469cb019872a0053ac791c4337902517f23 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp @@ -43,14 +43,11 @@ #include #include #include -#include #include #include namespace llvm { -class AssumptionCache; class DataLayout; -class DominatorTree; class LLVMContext; } // namespace llvm diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index 30e058000d2bc97c0a484cbe2d036aa37ffc4c87..f072f5cec3094ad80c30c0a20fc30b725aba052f 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -356,6 +356,7 @@ static bool simplifyAssocCastAssoc(BinaryOperator *BinOp1, IC.replaceOperand(*Cast, 0, BinOp2->getOperand(0)); IC.replaceOperand(*BinOp1, 1, FoldedC); + BinOp1->dropPoisonGeneratingFlags(); Cast->dropPoisonGeneratingFlags(); return true; } diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index da157c966bfcbed6a71a8750f5cbfe75b9c3a939..b175e6f93f3e8ff618c632bf247d056ac47acf20 100644 --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -2144,6 +2144,9 @@ ModuleAddressSanitizer::CreateMetadataGlobal(Module &M, Constant *Initializer, M, Initializer->getType(), false, Linkage, Initializer, Twine("__asan_global_") + GlobalValue::dropLLVMManglingEscape(OriginalName)); Metadata->setSection(getGlobalMetadataSection()); + // Place metadata in a large section for x86-64 ELF binaries to mitigate + // relocation pressure. + setGlobalVariableLargeSection(TargetTriple, *Metadata); return Metadata; } diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index 10258e25467913360e2921fdbeb9db58755b94f7..adb4ffd4c81272ef2fdb7d26f8606ef505775e96 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -49,6 +49,7 @@ #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/TargetParser/Triple.h" +#include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/ModuleUtils.h" @@ -406,13 +407,15 @@ enum class ValueProfilingCallType { } // end anonymous namespace -PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) { +PreservedAnalyses InstrProfilingLoweringPass::run(Module &M, + ModuleAnalysisManager &AM) { FunctionAnalysisManager &FAM = AM.getResult(M).getManager(); auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & { return FAM.getResult(F); }; - if (!run(M, GetTLI)) + InstrProfiling Lowerer(M, Options, GetTLI, IsCS); + if (!Lowerer.lower()) return PreservedAnalyses::all(); return PreservedAnalyses::none(); @@ -534,17 +537,7 @@ static bool containsProfilingIntrinsics(Module &M) { containsIntrinsic(llvm::Intrinsic::instrprof_value_profile); } -bool InstrProfiling::run( - Module &M, std::function GetTLI) { - this->M = &M; - this->GetTLI = std::move(GetTLI); - NamesVar = nullptr; - NamesSize = 0; - ProfileDataMap.clear(); - CompilerUsedVars.clear(); - UsedVars.clear(); - TT = Triple(M.getTargetTriple()); - +bool InstrProfiling::lower() { bool MadeChange = false; bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT); if (NeedsRuntimeHook) @@ -677,12 +670,12 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) { Ind->getOperandBundlesAsDefs(OpBundles); if (!IsMemOpSize) { Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(Index)}; - Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args, + Call = Builder.CreateCall(getOrInsertValueProfilingCall(M, *TLI), Args, OpBundles); } else { Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(Index)}; Call = Builder.CreateCall( - getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp), + getOrInsertValueProfilingCall(M, *TLI, ValueProfilingCallType::MemOp), Args, OpBundles); } if (auto AK = TLI->getExtAttrForI32Param(false)) @@ -704,18 +697,18 @@ Value *InstrProfiling::getCounterAddress(InstrProfCntrInstBase *I) { if (!isRuntimeCounterRelocationEnabled()) return Addr; - Type *Int64Ty = Type::getInt64Ty(M->getContext()); + Type *Int64Ty = Type::getInt64Ty(M.getContext()); Function *Fn = I->getParent()->getParent(); LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn]; if (!BiasLI) { IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front()); - auto *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName()); + auto *Bias = M.getGlobalVariable(getInstrProfCounterBiasVarName()); if (!Bias) { // Compiler must define this variable when runtime counter relocation // is being used. Runtime has a weak external reference that is used // to check whether that's the case or not. Bias = new GlobalVariable( - *M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage, + M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage, Constant::getNullValue(Int64Ty), getInstrProfCounterBiasVarName()); Bias->setVisibility(GlobalVariable::HiddenVisibility); // A definition that's weak (linkonce_odr) without being in a COMDAT @@ -723,7 +716,7 @@ Value *InstrProfiling::getCounterAddress(InstrProfCntrInstBase *I) { // data word from every TU but one. Putting it in COMDAT ensures there // will be exactly one data slot in the link. if (TT.supportsCOMDAT()) - Bias->setComdat(M->getOrInsertComdat(Bias->getName())); + Bias->setComdat(M.getOrInsertComdat(Bias->getName())); } BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias); } @@ -739,9 +732,9 @@ Value *InstrProfiling::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) { Bitmaps->getValueType(), Bitmaps, 0, I->getBitmapIndex()->getZExtValue()); if (isRuntimeCounterRelocationEnabled()) { - LLVMContext &Ctx = M->getContext(); + LLVMContext &Ctx = M.getContext(); Ctx.diagnose(DiagnosticInfoPGOProfile( - M->getName().data(), + M.getName().data(), Twine("Runtime counter relocation is presently not supported for MC/DC " "bitmaps."), DS_Warning)); @@ -762,12 +755,12 @@ void InstrProfiling::lowerTimestamp( InstrProfTimestampInst *TimestampInstruction) { assert(TimestampInstruction->getIndex()->isZeroValue() && "timestamp probes are always the first probe for a function"); - auto &Ctx = M->getContext(); + auto &Ctx = M.getContext(); auto *TimestampAddr = getCounterAddress(TimestampInstruction); IRBuilder<> Builder(TimestampInstruction); auto *CalleeTy = FunctionType::get(Type::getVoidTy(Ctx), TimestampAddr->getType(), false); - auto Callee = M->getOrInsertFunction( + auto Callee = M.getOrInsertFunction( INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), CalleeTy); Builder.CreateCall(Callee, {TimestampAddr}); TimestampInstruction->eraseFromParent(); @@ -812,10 +805,10 @@ void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) { void InstrProfiling::lowerMCDCTestVectorBitmapUpdate( InstrProfMCDCTVBitmapUpdate *Update) { IRBuilder<> Builder(Update); - auto *Int8Ty = Type::getInt8Ty(M->getContext()); - auto *Int8PtrTy = PointerType::getUnqual(M->getContext()); - auto *Int32Ty = Type::getInt32Ty(M->getContext()); - auto *Int64Ty = Type::getInt64Ty(M->getContext()); + auto *Int8Ty = Type::getInt8Ty(M.getContext()); + auto *Int8PtrTy = PointerType::getUnqual(M.getContext()); + auto *Int32Ty = Type::getInt32Ty(M.getContext()); + auto *Int64Ty = Type::getInt64Ty(M.getContext()); auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr(); auto *BitmapAddr = getBitmapAddress(Update); @@ -864,7 +857,7 @@ void InstrProfiling::lowerMCDCTestVectorBitmapUpdate( void InstrProfiling::lowerMCDCCondBitmapUpdate( InstrProfMCDCCondBitmapUpdate *Update) { IRBuilder<> Builder(Update); - auto *Int32Ty = Type::getInt32Ty(M->getContext()); + auto *Int32Ty = Type::getInt32Ty(M.getContext()); auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr(); // Load the MCDC temporary value from the stack. @@ -1046,8 +1039,8 @@ static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { void InstrProfiling::maybeSetComdat(GlobalVariable *GV, Function *Fn, StringRef VarName) { - bool DataReferencedByCode = profDataReferencedByCode(*M); - bool NeedComdat = needsComdatForCounter(*Fn, *M); + bool DataReferencedByCode = profDataReferencedByCode(M); + bool NeedComdat = needsComdatForCounter(*Fn, M); bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); if (!UseComdat) @@ -1055,7 +1048,7 @@ void InstrProfiling::maybeSetComdat(GlobalVariable *GV, Function *Fn, StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode ? GV->getName() : VarName; - Comdat *C = M->getOrInsertComdat(GroupName); + Comdat *C = M.getOrInsertComdat(GroupName); if (!NeedComdat) C->setSelectionKind(Comdat::NoDeduplicate); GV->setComdat(C); @@ -1140,8 +1133,8 @@ InstrProfiling::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, StringRef Name, GlobalValue::LinkageTypes Linkage) { uint64_t NumBytes = Inc->getNumBitmapBytes()->getZExtValue(); - auto *BitmapTy = ArrayType::get(Type::getInt8Ty(M->getContext()), NumBytes); - auto GV = new GlobalVariable(*M, BitmapTy, false, Linkage, + auto *BitmapTy = ArrayType::get(Type::getInt8Ty(M.getContext()), NumBytes); + auto GV = new GlobalVariable(M, BitmapTy, false, Linkage, Constant::getNullValue(BitmapTy), Name); GV->setAlignment(Align(1)); return GV; @@ -1166,7 +1159,7 @@ GlobalVariable * InstrProfiling::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name, GlobalValue::LinkageTypes Linkage) { uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); - auto &Ctx = M->getContext(); + auto &Ctx = M.getContext(); GlobalVariable *GV; if (isa(Inc)) { auto *CounterTy = Type::getInt8Ty(Ctx); @@ -1174,13 +1167,13 @@ InstrProfiling::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name, // TODO: `Constant::getAllOnesValue()` does not yet accept an array type. std::vector InitialValues(NumCounters, Constant::getAllOnesValue(CounterTy)); - GV = new GlobalVariable(*M, CounterArrTy, false, Linkage, + GV = new GlobalVariable(M, CounterArrTy, false, Linkage, ConstantArray::get(CounterArrTy, InitialValues), Name); GV->setAlignment(Align(1)); } else { auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); - GV = new GlobalVariable(*M, CounterTy, false, Linkage, + GV = new GlobalVariable(M, CounterTy, false, Linkage, Constant::getNullValue(CounterTy), Name); GV->setAlignment(Align(8)); } @@ -1200,10 +1193,10 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) { PD.RegionCounters = CounterPtr; if (DebugInfoCorrelate) { - LLVMContext &Ctx = M->getContext(); + LLVMContext &Ctx = M.getContext(); Function *Fn = Inc->getParent()->getParent(); if (auto *SP = Fn->getSubprogram()) { - DIBuilder DB(*M, true, SP->getUnit()); + DIBuilder DB(M, true, SP->getUnit()); Metadata *FunctionNameAnnotation[] = { MDString::get(Ctx, InstrProfCorrelator::FunctionNameAttributeName), MDString::get(Ctx, getPGOFuncNameVarInitializer(NamePtr)), @@ -1254,7 +1247,7 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc) { if (PD.DataVar) return; - LLVMContext &Ctx = M->getContext(); + LLVMContext &Ctx = M.getContext(); Function *Fn = Inc->getParent()->getParent(); GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage(); @@ -1270,8 +1263,8 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc) { Visibility = GlobalValue::DefaultVisibility; } - bool DataReferencedByCode = profDataReferencedByCode(*M); - bool NeedComdat = needsComdatForCounter(*Fn, *M); + bool DataReferencedByCode = profDataReferencedByCode(M); + bool NeedComdat = needsComdatForCounter(*Fn, M); bool Renamed; // The Data Variable section is anchored to profile counters. @@ -1291,9 +1284,10 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc) { !needsRuntimeRegistrationOfSectionRange(TT)) { ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS); auto *ValuesVar = new GlobalVariable( - *M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy), + M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy), getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed)); ValuesVar->setVisibility(Visibility); + setGlobalVariableLargeSection(TT, *ValuesVar); ValuesVar->setSection( getInstrProfSectionName(IPSK_vals, TT.getObjectFormat())); ValuesVar->setAlignment(Align(8)); @@ -1307,7 +1301,7 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc) { uint64_t NumBitmapBytes = PD.NumBitmapBytes; // Create data variable. - auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext()); + auto *IntPtrTy = M.getDataLayout().getIntPtrType(M.getContext()); auto *Int16Ty = Type::getInt16Ty(Ctx); auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1); Type *DataTypes[] = { @@ -1340,7 +1334,7 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc) { Visibility = GlobalValue::DefaultVisibility; } auto *Data = - new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName); + new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName); // Reference the counter variable with a label difference (link-time // constant). auto *RelativeCounterPtr = @@ -1411,7 +1405,7 @@ void InstrProfiling::emitVNodes() { if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS) NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2); - auto &Ctx = M->getContext(); + auto &Ctx = M.getContext(); Type *VNodeTypes[] = { #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType, #include "llvm/ProfileData/InstrProfData.inc" @@ -1420,11 +1414,12 @@ void InstrProfiling::emitVNodes() { ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters); auto *VNodesVar = new GlobalVariable( - *M, VNodesTy, false, GlobalValue::PrivateLinkage, + M, VNodesTy, false, GlobalValue::PrivateLinkage, Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName()); + setGlobalVariableLargeSection(TT, *VNodesVar); VNodesVar->setSection( getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat())); - VNodesVar->setAlignment(M->getDataLayout().getABITypeAlign(VNodesTy)); + VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(VNodesTy)); // VNodesVar is used by runtime but not referenced via relocation by other // sections. Conservatively make it linker retained. UsedVars.push_back(VNodesVar); @@ -1442,13 +1437,14 @@ void InstrProfiling::emitNameData() { report_fatal_error(Twine(toString(std::move(E))), false); } - auto &Ctx = M->getContext(); + auto &Ctx = M.getContext(); auto *NamesVal = ConstantDataArray::getString(Ctx, StringRef(CompressedNameStr), false); - NamesVar = new GlobalVariable(*M, NamesVal->getType(), true, + NamesVar = new GlobalVariable(M, NamesVal->getType(), true, GlobalValue::PrivateLinkage, NamesVal, getInstrProfNamesVarName()); NamesSize = CompressedNameStr.size(); + setGlobalVariableLargeSection(TT, *NamesVar); NamesVar->setSection( getInstrProfSectionName(IPSK_name, TT.getObjectFormat())); // On COFF, it's important to reduce the alignment down to 1 to prevent the @@ -1468,9 +1464,9 @@ void InstrProfiling::emitRegistration() { return; // Construct the function. - auto *VoidTy = Type::getVoidTy(M->getContext()); - auto *VoidPtrTy = PointerType::getUnqual(M->getContext()); - auto *Int64Ty = Type::getInt64Ty(M->getContext()); + auto *VoidTy = Type::getVoidTy(M.getContext()); + auto *VoidPtrTy = PointerType::getUnqual(M.getContext()); + auto *Int64Ty = Type::getInt64Ty(M.getContext()); auto *RegisterFTy = FunctionType::get(VoidTy, false); auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage, getInstrProfRegFuncsName(), M); @@ -1483,7 +1479,7 @@ void InstrProfiling::emitRegistration() { Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage, getInstrProfRegFuncName(), M); - IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF)); + IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", RegisterF)); for (Value *Data : CompilerUsedVars) if (!isa(Data)) IRB.CreateCall(RuntimeRegisterF, Data); @@ -1511,13 +1507,13 @@ bool InstrProfiling::emitRuntimeHook() { return false; // If the module's provided its own runtime, we don't need to do anything. - if (M->getGlobalVariable(getInstrProfRuntimeHookVarName())) + if (M.getGlobalVariable(getInstrProfRuntimeHookVarName())) return false; // Declare an external variable that will pull in the runtime initialization. - auto *Int32Ty = Type::getInt32Ty(M->getContext()); + auto *Int32Ty = Type::getInt32Ty(M.getContext()); auto *Var = - new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage, + new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage, nullptr, getInstrProfRuntimeHookVarName()); Var->setVisibility(GlobalValue::HiddenVisibility); @@ -1534,9 +1530,9 @@ bool InstrProfiling::emitRuntimeHook() { User->addFnAttr(Attribute::NoRedZone); User->setVisibility(GlobalValue::HiddenVisibility); if (TT.supportsCOMDAT()) - User->setComdat(M->getOrInsertComdat(User->getName())); + User->setComdat(M.getOrInsertComdat(User->getName())); - IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User)); + IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", User)); auto *Load = IRB.CreateLoad(Int32Ty, Var); IRB.CreateRet(Load); @@ -1557,15 +1553,15 @@ void InstrProfiling::emitUses() { // and ensure this GC property as well. Otherwise, we have to conservatively // make all of the sections retained by the linker. if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() || - (TT.isOSBinFormatCOFF() && !profDataReferencedByCode(*M))) - appendToCompilerUsed(*M, CompilerUsedVars); + (TT.isOSBinFormatCOFF() && !profDataReferencedByCode(M))) + appendToCompilerUsed(M, CompilerUsedVars); else - appendToUsed(*M, CompilerUsedVars); + appendToUsed(M, CompilerUsedVars); // We do not add proper references from used metadata sections to NamesVar and // VNodesVar, so we have to be conservative and place them in llvm.used // regardless of the target, - appendToUsed(*M, UsedVars); + appendToUsed(M, UsedVars); } void InstrProfiling::emitInitialization() { @@ -1574,13 +1570,13 @@ void InstrProfiling::emitInitialization() { // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should // have already create the variable before LTO/ThinLTO linking. if (!IsCS) - createProfileFileNameVar(*M, Options.InstrProfileOutput); - Function *RegisterF = M->getFunction(getInstrProfRegFuncsName()); + createProfileFileNameVar(M, Options.InstrProfileOutput); + Function *RegisterF = M.getFunction(getInstrProfRegFuncsName()); if (!RegisterF) return; // Create the initialization function. - auto *VoidTy = Type::getVoidTy(M->getContext()); + auto *VoidTy = Type::getVoidTy(M.getContext()); auto *F = Function::Create(FunctionType::get(VoidTy, false), GlobalValue::InternalLinkage, getInstrProfInitFuncName(), M); @@ -1590,9 +1586,9 @@ void InstrProfiling::emitInitialization() { F->addFnAttr(Attribute::NoRedZone); // Add the basic block and the necessary calls. - IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F)); + IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", F)); IRB.CreateCall(RegisterF, {}); IRB.CreateRetVoid(); - appendToGlobalCtors(*M, F, 0); + appendToGlobalCtors(M, F, 0); } diff --git a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp index 806afc8fcdf7cb327adee47ce52a4c58183ca3e5..7a03ee46d6fded1d2a4aaf8f94cf0de998c041bf 100644 --- a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp @@ -85,3 +85,10 @@ Comdat *llvm::getOrCreateFunctionComdat(Function &F, Triple &T) { return C; } +void llvm::setGlobalVariableLargeSection(const Triple &TargetTriple, + GlobalVariable &GV) { + if (TargetTriple.getArch() == Triple::x86_64 && + TargetTriple.getObjectFormat() == Triple::ELF) { + GV.setCodeModel(CodeModel::Large); + } +} diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp index 9066876635191222916cb01f9ea5817d564e1974..fe672a4377a1f4e85b948f998e11afb380363961 100644 --- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp +++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp @@ -329,8 +329,7 @@ ModuleSanitizerCoverage::CreateSecStartEnd(Module &M, const char *Section, // Account for the fact that on windows-msvc __start_* symbols actually // point to a uint64_t before the start of the array. - auto SecStartI8Ptr = IRB.CreatePointerCast(SecStart, PtrTy); - auto GEP = IRB.CreateGEP(Int8Ty, SecStartI8Ptr, + auto GEP = IRB.CreateGEP(Int8Ty, SecStart, ConstantInt::get(IntptrTy, sizeof(uint64_t))); return std::make_pair(GEP, SecEnd); } @@ -838,8 +837,7 @@ void ModuleSanitizerCoverage::InjectTraceForSwitch( *CurModule, ArrayOfInt64Ty, false, GlobalVariable::InternalLinkage, ConstantArray::get(ArrayOfInt64Ty, Initializers), "__sancov_gen_cov_switch_values"); - IRB.CreateCall(SanCovTraceSwitchFunction, - {Cond, IRB.CreatePointerCast(GV, PtrTy)}); + IRB.CreateCall(SanCovTraceSwitchFunction, {Cond, GV}); } } } diff --git a/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp b/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp index 63b7903ef955d9b357da5f7e18e8c1ae728a22ea..f3422a705dca7a86189e879418831e72e2abe8e3 100644 --- a/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp +++ b/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp @@ -83,11 +83,7 @@ static Align getNewAlignment(const SCEV *AASCEV, const SCEV *AlignSCEV, const SCEV *OffSCEV, Value *Ptr, ScalarEvolution *SE) { const SCEV *PtrSCEV = SE->getSCEV(Ptr); - // On a platform with 32-bit allocas, but 64-bit flat/global pointer sizes - // (*cough* AMDGPU), the effective SCEV type of AASCEV and PtrSCEV - // may disagree. Trunc/extend so they agree. - PtrSCEV = SE->getTruncateOrZeroExtend( - PtrSCEV, SE->getEffectiveSCEVType(AASCEV->getType())); + const SCEV *DiffSCEV = SE->getMinusSCEV(PtrSCEV, AASCEV); if (isa(DiffSCEV)) return Align(1); @@ -267,11 +263,17 @@ bool AlignmentFromAssumptionsPass::processAssumption(CallInst *ACall, // Now that we've updated that use of the pointer, look for other uses of // the pointer to update. Visited.insert(J); - for (User *UJ : J->users()) { - Instruction *K = cast(UJ); - if (!Visited.count(K)) - WorkList.push_back(K); - } + if (isa(J) || isa(J)) + for (auto &U : J->uses()) { + if (U->getType()->isPointerTy()) { + Instruction *K = cast(U.getUser()); + StoreInst *SI = dyn_cast(K); + if (SI && SI->getPointerOperandIndex() != U.getOperandNo()) + continue; + if (!Visited.count(K)) + WorkList.push_back(K); + } + } } return true; diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp index a6fbddca5cba4df9a881a23de92f735b63b84c56..43d9883fcfd4868e9967c645e3eb260a1f125af7 100644 --- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp +++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp @@ -26,7 +26,6 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" -#include "llvm/IR/GetElementPtrTypeIterator.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instructions.h" @@ -36,7 +35,6 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/DebugCounter.h" -#include "llvm/Support/KnownBits.h" #include "llvm/Support/MathExtras.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/ValueMapper.h" diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp index 5f82af1ca46d3c753f254882898cc208e3ffe686..9df28747570c4db4552f110b1efa17f333b7ebaa 100644 --- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp +++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp @@ -89,7 +89,6 @@ #include #include #include -#include #include #include diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp index 446aa497026d3fbfec2538ddb13f318b383778c2..f14541a1a037e6da98136a091c61c6a91acc5ae8 100644 --- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp +++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp @@ -755,7 +755,7 @@ static unsigned getFullUnrollBoostingFactor(const EstimatedUnrollCost &Cost, static std::optional shouldPragmaUnroll(Loop *L, const PragmaInfo &PInfo, const unsigned TripMultiple, const unsigned TripCount, - const UnrollCostEstimator UCE, + unsigned MaxTripCount, const UnrollCostEstimator UCE, const TargetTransformInfo::UnrollingPreferences &UP) { // Using unroll pragma @@ -776,6 +776,10 @@ shouldPragmaUnroll(Loop *L, const PragmaInfo &PInfo, if (PInfo.PragmaFullUnroll && TripCount != 0) return TripCount; + if (PInfo.PragmaEnableUnroll && !TripCount && MaxTripCount && + MaxTripCount <= UnrollMaxUpperBound) + return MaxTripCount; + // if didn't return until here, should continue to other priorties return std::nullopt; } @@ -902,7 +906,7 @@ bool llvm::computeUnrollCount( // 1st priority is unroll count set by "unroll-count" option. // 2nd priority is unroll count set by pragma. if (auto UnrollFactor = shouldPragmaUnroll(L, PInfo, TripMultiple, TripCount, - UCE, UP)) { + MaxTripCount, UCE, UP)) { UP.Count = *UnrollFactor; if (UserUnrollCount || (PragmaCount > 0)) { diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp index 0e55249d63a82947c7476868c731ef994daeae97..9d058e0d248378068d3cc03145d6727bd40b95df 100644 --- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp +++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp @@ -880,8 +880,11 @@ bool MemCpyOptPass::performCallSlotOptzn(Instruction *cpyLoad, return false; const DataLayout &DL = cpyLoad->getModule()->getDataLayout(); - uint64_t srcSize = DL.getTypeAllocSize(srcAlloca->getAllocatedType()) * - srcArraySize->getZExtValue(); + TypeSize SrcAllocaSize = DL.getTypeAllocSize(srcAlloca->getAllocatedType()); + // We can't optimize scalable types. + if (SrcAllocaSize.isScalable()) + return false; + uint64_t srcSize = SrcAllocaSize * srcArraySize->getZExtValue(); if (cpySize < srcSize) return false; diff --git a/llvm/lib/Transforms/Scalar/Reassociate.cpp b/llvm/lib/Transforms/Scalar/Reassociate.cpp index 42e979db24d29b6107e1b8f3d9104721ee7b6a44..818c7b40d489efe88c04e8011a75621e78bbc37e 100644 --- a/llvm/lib/Transforms/Scalar/Reassociate.cpp +++ b/llvm/lib/Transforms/Scalar/Reassociate.cpp @@ -466,7 +466,8 @@ using RepeatedValue = std::pair; /// type and thus make the expression bigger. static bool LinearizeExprTree(Instruction *I, SmallVectorImpl &Ops, - ReassociatePass::OrderedSet &ToRedo) { + ReassociatePass::OrderedSet &ToRedo, + bool &HasNUW) { assert((isa(I) || isa(I)) && "Expected a UnaryOperator or BinaryOperator!"); LLVM_DEBUG(dbgs() << "LINEARIZE: " << *I << '\n'); @@ -515,6 +516,9 @@ static bool LinearizeExprTree(Instruction *I, std::pair P = Worklist.pop_back_val(); I = P.first; // We examine the operands of this binary operator. + if (isa(I)) + HasNUW &= I->hasNoUnsignedWrap(); + for (unsigned OpIdx = 0; OpIdx < I->getNumOperands(); ++OpIdx) { // Visit operands. Value *Op = I->getOperand(OpIdx); APInt Weight = P.second; // Number of paths to this operand. @@ -657,7 +661,8 @@ static bool LinearizeExprTree(Instruction *I, /// Now that the operands for this expression tree are /// linearized and optimized, emit them in-order. void ReassociatePass::RewriteExprTree(BinaryOperator *I, - SmallVectorImpl &Ops) { + SmallVectorImpl &Ops, + bool HasNUW) { assert(Ops.size() > 1 && "Single values should be used directly!"); // Since our optimizations should never increase the number of operations, the @@ -814,14 +819,20 @@ void ReassociatePass::RewriteExprTree(BinaryOperator *I, if (ExpressionChangedStart) { bool ClearFlags = true; do { - // Preserve FastMathFlags. + // Preserve flags. if (ClearFlags) { if (isa(I)) { FastMathFlags Flags = I->getFastMathFlags(); ExpressionChangedStart->clearSubclassOptionalData(); ExpressionChangedStart->setFastMathFlags(Flags); - } else + } else { ExpressionChangedStart->clearSubclassOptionalData(); + // Note that it doesn't hold for mul if one of the operands is zero. + // TODO: We can preserve NUW flag if we prove that all mul operands + // are non-zero. + if (HasNUW && ExpressionChangedStart->getOpcode() == Instruction::Add) + ExpressionChangedStart->setHasNoUnsignedWrap(); + } } if (ExpressionChangedStart == ExpressionChangedEnd) @@ -1175,7 +1186,8 @@ Value *ReassociatePass::RemoveFactorFromExpression(Value *V, Value *Factor) { return nullptr; SmallVector Tree; - MadeChange |= LinearizeExprTree(BO, Tree, RedoInsts); + bool HasNUW = true; + MadeChange |= LinearizeExprTree(BO, Tree, RedoInsts, HasNUW); SmallVector Factors; Factors.reserve(Tree.size()); for (unsigned i = 0, e = Tree.size(); i != e; ++i) { @@ -1217,7 +1229,7 @@ Value *ReassociatePass::RemoveFactorFromExpression(Value *V, Value *Factor) { if (!FoundFactor) { // Make sure to restore the operands to the expression tree. - RewriteExprTree(BO, Factors); + RewriteExprTree(BO, Factors, HasNUW); return nullptr; } @@ -1229,7 +1241,7 @@ Value *ReassociatePass::RemoveFactorFromExpression(Value *V, Value *Factor) { RedoInsts.insert(BO); V = Factors[0].Op; } else { - RewriteExprTree(BO, Factors); + RewriteExprTree(BO, Factors, HasNUW); V = BO; } @@ -2354,7 +2366,8 @@ void ReassociatePass::ReassociateExpression(BinaryOperator *I) { // First, walk the expression tree, linearizing the tree, collecting the // operand information. SmallVector Tree; - MadeChange |= LinearizeExprTree(I, Tree, RedoInsts); + bool HasNUW = true; + MadeChange |= LinearizeExprTree(I, Tree, RedoInsts, HasNUW); SmallVector Ops; Ops.reserve(Tree.size()); for (const RepeatedValue &E : Tree) @@ -2547,7 +2560,7 @@ void ReassociatePass::ReassociateExpression(BinaryOperator *I) { dbgs() << '\n'); // Now that we ordered and optimized the expressions, splat them back into // the expression tree, removing any unneeded nodes. - RewriteExprTree(I, Ops); + RewriteExprTree(I, Ops, HasNUW); } void diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp index 69679b608f8d71e988188b7cba02a7510a9fd153..8a491e74b91c623f413063783f3522436751b15f 100644 --- a/llvm/lib/Transforms/Scalar/SCCP.cpp +++ b/llvm/lib/Transforms/Scalar/SCCP.cpp @@ -46,7 +46,6 @@ #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/SCCPSolver.h" -#include #include using namespace llvm; diff --git a/llvm/lib/Transforms/Utils/LowerSwitch.cpp b/llvm/lib/Transforms/Utils/LowerSwitch.cpp index 227de425ff855498196ce756774360e75e65ef3f..d1cdab7599c4985136b7c2b075eac89fc7f3a178 100644 --- a/llvm/lib/Transforms/Utils/LowerSwitch.cpp +++ b/llvm/lib/Transforms/Utils/LowerSwitch.cpp @@ -42,7 +42,6 @@ #include #include #include -#include #include using namespace llvm; diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index 8bfe9e67d15e2f95f164f28a81717cb1d915f9f8..cd3ac317cd238e2f67e0580f8fccc10c06ca5ec4 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -1532,8 +1532,26 @@ Value *SCEVExpander::expand(const SCEV *S) { V = visit(S); V = fixupLCSSAFormFor(V); } else { - for (Instruction *I : DropPoisonGeneratingInsts) + for (Instruction *I : DropPoisonGeneratingInsts) { I->dropPoisonGeneratingFlagsAndMetadata(); + // See if we can re-infer from first principles any of the flags we just + // dropped. + if (auto *OBO = dyn_cast(I)) + if (auto Flags = SE.getStrengthenedNoWrapFlagsFromBinOp(OBO)) { + auto *BO = cast(I); + BO->setHasNoUnsignedWrap( + ScalarEvolution::maskFlags(*Flags, SCEV::FlagNUW) == SCEV::FlagNUW); + BO->setHasNoSignedWrap( + ScalarEvolution::maskFlags(*Flags, SCEV::FlagNSW) == SCEV::FlagNSW); + } + if (auto *NNI = dyn_cast(I)) { + auto *Src = NNI->getOperand(0); + if (isImpliedByDomCondition(ICmpInst::ICMP_SGE, Src, + Constant::getNullValue(Src->getType()), I, + DL).value_or(false)) + NNI->setNonNeg(true); + } + } } // Remember the expanded value for this SCEV at this location. // diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index c09cf9c2325c405e0d1d6de4572c1ef829fa14fc..89494a7f64971fd0e4fda8b0af118af0323e3c9f 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -15,7 +15,6 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/Sequence.h" #include "llvm/ADT/SetOperations.h" #include "llvm/ADT/SetVector.h" diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp index c0dbd52acbabe5b6656aacb65970d56255d6666e..fa2459d1ca02879af30315654d58499b80ccb170 100644 --- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp @@ -108,7 +108,6 @@ #include #include #include -#include #include #include #include diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index c07e9f043b7b89a1d15fe1a6c45ca9a802aadd8d..f82e161fb846d18e86af07dcc1bbcff7dbad108d 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -6888,6 +6888,30 @@ void LoopVectorizationCostModel::setVectorizedCallDecision(ElementCount VF) { ParamsOk = false; break; } + case VFParamKind::OMP_Linear: { + Value *ScalarParam = CI->getArgOperand(Param.ParamPos); + // Find the stride for the scalar parameter in this loop and see if + // it matches the stride for the variant. + // TODO: do we need to figure out the cost of an extract to get the + // first lane? Or do we hope that it will be folded away? + ScalarEvolution *SE = PSE.getSE(); + const auto *SAR = + dyn_cast(SE->getSCEV(ScalarParam)); + + if (!SAR || SAR->getLoop() != TheLoop) { + ParamsOk = false; + break; + } + + const SCEVConstant *Step = + dyn_cast(SAR->getStepRecurrence(*SE)); + + if (!Step || + Step->getAPInt().getSExtValue() != Param.LinearStepOrPos) + ParamsOk = false; + + break; + } case VFParamKind::GlobalPredicate: UsesMask = true; break; @@ -8621,7 +8645,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW, // Add a CanonicalIVIncrement{NUW} VPInstruction to increment the scalar // IV by VF * UF. auto *CanonicalIVIncrement = - new VPInstruction(VPInstruction::CanonicalIVIncrement, {CanonicalIVPHI}, + new VPInstruction(Instruction::Add, {CanonicalIVPHI, &Plan.getVFxUF()}, {HasNUW, false}, DL, "index.next"); CanonicalIVPHI->addOperand(CanonicalIVIncrement); diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index a1e3e76ac852aae3a054c758e777e02ca89a8429..fe2aac78e5ab0debb427b4bfbdeec54873e62312 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -2883,6 +2883,10 @@ private: assert(!BundleMember && "Bundle and VL out of sync"); } else { MustGather.insert(VL.begin(), VL.end()); + // Build a map for gathered scalars to the nodes where they are used. + for (Value *V : VL) + if (!isConstant(V)) + ValueToGatherNodes.try_emplace(V).first->getSecond().insert(Last); } if (UserTreeIdx.UserTE) @@ -3754,7 +3758,40 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE) { OrdersType CurrentOrder(NumScalars, NumScalars); SmallVector Positions; SmallBitVector UsedPositions(NumScalars); - const TreeEntry *STE = nullptr; + DenseMap UsedEntries; + DenseMap> ValueToEntryPos; + for (Value *V : TE.Scalars) { + if (!isa(V)) + continue; + const auto *LocalSTE = getTreeEntry(V); + if (!LocalSTE) + continue; + unsigned Lane = + std::distance(LocalSTE->Scalars.begin(), find(LocalSTE->Scalars, V)); + if (Lane >= NumScalars) + continue; + ++UsedEntries.try_emplace(LocalSTE, 0).first->getSecond(); + ValueToEntryPos.try_emplace(V, LocalSTE, Lane); + } + if (UsedEntries.empty()) + return std::nullopt; + const TreeEntry &BestSTE = + *std::max_element(UsedEntries.begin(), UsedEntries.end(), + [](const std::pair &P1, + const std::pair &P2) { + return P1.second < P2.second; + }) + ->first; + UsedEntries.erase(&BestSTE); + const TreeEntry *SecondBestSTE = nullptr; + if (!UsedEntries.empty()) + SecondBestSTE = + std::max_element(UsedEntries.begin(), UsedEntries.end(), + [](const std::pair &P1, + const std::pair &P2) { + return P1.second < P2.second; + }) + ->first; // Try to find all gathered scalars that are gets vectorized in other // vectorize node. Here we can have only one single tree vector node to // correctly identify order of the gathered scalars. @@ -3762,53 +3799,46 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE) { Value *V = TE.Scalars[I]; if (!isa(V)) continue; - if (const auto *LocalSTE = getTreeEntry(V)) { - if (!STE) - STE = LocalSTE; - else if (STE != LocalSTE) - // Take the order only from the single vector node. - return std::nullopt; - unsigned Lane = - std::distance(STE->Scalars.begin(), find(STE->Scalars, V)); - if (Lane >= NumScalars) - return std::nullopt; - if (CurrentOrder[Lane] != NumScalars) { - if (Lane != I) - continue; - UsedPositions.reset(CurrentOrder[Lane]); - } - // The partial identity (where only some elements of the gather node are - // in the identity order) is good. - CurrentOrder[Lane] = I; - UsedPositions.set(I); + const auto [LocalSTE, Lane] = ValueToEntryPos.lookup(V); + if (!LocalSTE || (LocalSTE != &BestSTE && LocalSTE != SecondBestSTE)) + continue; + if (CurrentOrder[Lane] != NumScalars) { + if ((CurrentOrder[Lane] >= BestSTE.Scalars.size() || + BestSTE.Scalars[CurrentOrder[Lane]] == V) && + (Lane != I || LocalSTE == SecondBestSTE)) + continue; + UsedPositions.reset(CurrentOrder[Lane]); } + // The partial identity (where only some elements of the gather node are + // in the identity order) is good. + CurrentOrder[Lane] = I; + UsedPositions.set(I); } // Need to keep the order if we have a vector entry and at least 2 scalars or // the vectorized entry has just 2 scalars. - if (STE && (UsedPositions.count() > 1 || STE->Scalars.size() == 2)) { - auto &&IsIdentityOrder = [NumScalars](ArrayRef CurrentOrder) { - for (unsigned I = 0; I < NumScalars; ++I) - if (CurrentOrder[I] != I && CurrentOrder[I] != NumScalars) - return false; - return true; - }; - if (IsIdentityOrder(CurrentOrder)) - return OrdersType(); - auto *It = CurrentOrder.begin(); - for (unsigned I = 0; I < NumScalars;) { - if (UsedPositions.test(I)) { - ++I; - continue; - } - if (*It == NumScalars) { - *It = I; - ++I; - } - ++It; + if (BestSTE.Scalars.size() != 2 && UsedPositions.count() <= 1) + return std::nullopt; + auto IsIdentityOrder = [&](ArrayRef CurrentOrder) { + for (unsigned I = 0; I < NumScalars; ++I) + if (CurrentOrder[I] != I && CurrentOrder[I] != NumScalars) + return false; + return true; + }; + if (IsIdentityOrder(CurrentOrder)) + return OrdersType(); + auto *It = CurrentOrder.begin(); + for (unsigned I = 0; I < NumScalars;) { + if (UsedPositions.test(I)) { + ++I; + continue; + } + if (*It == NumScalars) { + *It = I; + ++I; } - return std::move(CurrentOrder); + ++It; } - return std::nullopt; + return std::move(CurrentOrder); } namespace { @@ -5597,6 +5627,18 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, InstructionsState S = getSameOpcode(VL, *TLI); + // Don't vectorize ephemeral values. + if (!EphValues.empty()) { + for (Value *V : VL) { + if (EphValues.count(V)) { + LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V + << ") is ephemeral.\n"); + newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx); + return; + } + } + } + // Gather if we hit the RecursionMaxDepth, unless this is a load (or z/sext of // a load), in which case peek through to include it in the tree, without // ballooning over-budget. @@ -5735,18 +5777,6 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, // We now know that this is a vector of instructions of the same type from // the same block. - // Don't vectorize ephemeral values. - if (!EphValues.empty()) { - for (Value *V : VL) { - if (EphValues.count(V)) { - LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V - << ") is ephemeral.\n"); - newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx); - return; - } - } - } - // Check if this is a duplicate of another entry. if (TreeEntry *E = getTreeEntry(S.OpValue)) { LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n"); @@ -8865,16 +8895,6 @@ static T *performExtractsShuffleAction( } InstructionCost BoUpSLP::getTreeCost(ArrayRef VectorizedVals) { - // Build a map for gathered scalars to the nodes where they are used. - ValueToGatherNodes.clear(); - for (const std::unique_ptr &EntryPtr : VectorizableTree) { - if (EntryPtr->State != TreeEntry::NeedToGather) - continue; - for (Value *V : EntryPtr->Scalars) - if (!isConstant(V)) - ValueToGatherNodes.try_emplace(V).first->getSecond().insert( - EntryPtr.get()); - } InstructionCost Cost = 0; LLVM_DEBUG(dbgs() << "SLP: Calculating cost for tree of size " << VectorizableTree.size() << ".\n"); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 0074532f9c68d7b7201301da4052b63ffebb6e68..263d9938d1f0f4a5e0f529f2a84e557fa564c887 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -741,6 +741,12 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV, for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) State.set(&VectorTripCount, VectorTripCountV, Part); + IRBuilder<> Builder(State.CFG.PrevBB->getTerminator()); + // FIXME: Model VF * UF computation completely in VPlan. + State.set(&VFxUF, + createStepForVF(Builder, TripCountV->getType(), State.VF, State.UF), + 0); + // When vectorizing the epilogue loop, the canonical induction start value // needs to be changed from zero to the value after the main vector loop. // FIXME: Improve modeling for canonical IV start values in the epilogue loop. @@ -752,7 +758,7 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV, return isa(U) || isa(U) || cast(U)->getOpcode() == - VPInstruction::CanonicalIVIncrement; + Instruction::Add; }) && "the canonical IV should only be used by its increment or " "ScalarIVSteps when resetting the start value"); @@ -845,6 +851,13 @@ void VPlan::execute(VPTransformState *State) { #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPlan::printLiveIns(raw_ostream &O) const { VPSlotTracker SlotTracker(this); + + if (VFxUF.getNumUsers() > 0) { + O << "\nLive-in "; + VFxUF.printAsOperand(O, SlotTracker); + O << " = VF * UF"; + } + if (VectorTripCount.getNumUsers() > 0) { O << "\nLive-in "; VectorTripCount.printAsOperand(O, SlotTracker); @@ -1237,6 +1250,8 @@ void VPSlotTracker::assignSlot(const VPValue *V) { } void VPSlotTracker::assignSlots(const VPlan &Plan) { + if (Plan.VFxUF.getNumUsers() > 0) + assignSlot(&Plan.VFxUF); assignSlot(&Plan.VectorTripCount); if (Plan.BackedgeTakenCount) assignSlot(Plan.BackedgeTakenCount); @@ -1260,6 +1275,11 @@ bool vputils::onlyFirstLaneUsed(VPValue *Def) { [Def](VPUser *U) { return U->onlyFirstLaneUsed(Def); }); } +bool vputils::onlyFirstPartUsed(VPValue *Def) { + return all_of(Def->users(), + [Def](VPUser *U) { return U->onlyFirstPartUsed(Def); }); +} + VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE) { if (auto *Expanded = Plan.getSCEVExpansion(Expr)) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 9002452204480ef265a8b5c2bc6ea13c4b1a6804..94cb768898136140e370646ff3b57398e71b91e3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1058,9 +1058,7 @@ public: SLPStore, ActiveLaneMask, CalculateTripCountMinusVF, - CanonicalIVIncrement, - // The next op is similar to the above, but instead increment the - // canonical IV separately for each unrolled part. + // Increment the canonical IV separately for each unrolled part. CanonicalIVIncrementForPart, BranchOnCount, BranchOnCond @@ -1168,13 +1166,27 @@ public: return false; case VPInstruction::ActiveLaneMask: case VPInstruction::CalculateTripCountMinusVF: - case VPInstruction::CanonicalIVIncrement: case VPInstruction::CanonicalIVIncrementForPart: case VPInstruction::BranchOnCount: return true; }; llvm_unreachable("switch should return"); } + + /// Returns true if the recipe only uses the first part of operand \p Op. + bool onlyFirstPartUsed(const VPValue *Op) const override { + assert(is_contained(operands(), Op) && + "Op must be an operand of the recipe"); + if (getOperand(0) != Op) + return false; + switch (getOpcode()) { + default: + return false; + case VPInstruction::BranchOnCount: + return true; + }; + llvm_unreachable("switch should return"); + } }; /// VPWidenRecipe is a recipe for producing a copy of vector type its @@ -2126,6 +2138,13 @@ public: return true; } + /// Returns true if the recipe only uses the first part of operand \p Op. + bool onlyFirstPartUsed(const VPValue *Op) const override { + assert(is_contained(operands(), Op) && + "Op must be an operand of the recipe"); + return true; + } + /// Check if the induction described by \p Kind, /p Start and \p Step is /// canonical, i.e. has the same start, step (of 1), and type as the /// canonical IV. @@ -2545,6 +2564,9 @@ class VPlan { /// Represents the vector trip count. VPValue VectorTripCount; + /// Represents the loop-invariant VF * UF of the vector loop region. + VPValue VFxUF; + /// Holds a mapping between Values and their corresponding VPValue inside /// VPlan. Value2VPValueTy Value2VPValue; @@ -2624,6 +2646,9 @@ public: /// The vector trip count. VPValue &getVectorTripCount() { return VectorTripCount; } + /// Returns VF * UF of the vector loop region. + VPValue &getVFxUF() { return VFxUF; } + /// Mark the plan to indicate that using Value2VPValue is not safe any /// longer, because it may be stale. void disableValue2VPValue() { Value2VPValueEnabled = false; } @@ -3054,6 +3079,9 @@ namespace vputils { /// Returns true if only the first lane of \p Def is used. bool onlyFirstLaneUsed(VPValue *Def); +/// Returns true if only the first part of \p Def is used. +bool onlyFirstPartUsed(VPValue *Def); + /// Get or create a VPValue that corresponds to the expansion of \p Expr. If \p /// Expr is a SCEVConstant or SCEVUnknown, return a VPValue wrapping the live-in /// value. Otherwise return a VPExpandSCEVRecipe to expand \p Expr. If \p Plan's diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h index 473a7c28e48af88168802131d3eb6063d14c69fc..7276641551ae80ff4910c13478ea0b1fba52e6c7 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h @@ -16,12 +16,9 @@ namespace llvm { class LLVMContext; class VPValue; class VPBlendRecipe; -class VPInterleaveRecipe; class VPInstruction; -class VPReductionPHIRecipe; class VPWidenRecipe; class VPWidenCallRecipe; -class VPWidenCastRecipe; class VPWidenIntOrFpInductionRecipe; class VPWidenMemoryInstructionRecipe; struct VPWidenSelectRecipe; diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 1891b211a3566bc3b9e0c07dd1a8fc355e2af18b..ffdeea80fa074b8b14e964c1c52f3b8bea26eb28 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -118,9 +118,9 @@ bool VPRecipeBase::mayHaveSideEffects() const { case VPInstructionSC: switch (cast(this)->getOpcode()) { case Instruction::ICmp: + case Instruction::Select: case VPInstruction::Not: case VPInstruction::CalculateTripCountMinusVF: - case VPInstruction::CanonicalIVIncrement: case VPInstruction::CanonicalIVIncrementForPart: return false; default: @@ -272,9 +272,16 @@ Value *VPInstruction::generateInstruction(VPTransformState &State, Builder.SetCurrentDebugLocation(getDebugLoc()); if (Instruction::isBinaryOp(getOpcode())) { + if (Part != 0 && vputils::onlyFirstPartUsed(this)) + return State.get(this, 0); + Value *A = State.get(getOperand(0), Part); Value *B = State.get(getOperand(1), Part); - return Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name); + auto *Res = + Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name); + if (auto *I = dyn_cast(Res)) + setFlags(I); + return Res; } switch (getOpcode()) { @@ -335,19 +342,6 @@ Value *VPInstruction::generateInstruction(VPTransformState &State, Value *Zero = ConstantInt::get(ScalarTC->getType(), 0); return Builder.CreateSelect(Cmp, Sub, Zero); } - case VPInstruction::CanonicalIVIncrement: { - if (Part == 0) { - auto *Phi = State.get(getOperand(0), 0); - // The loop step is equal to the vectorization factor (num of SIMD - // elements) times the unroll factor (num of SIMD instructions). - Value *Step = - createStepForVF(Builder, Phi->getType(), State.VF, State.UF); - return Builder.CreateAdd(Phi, Step, Name, hasNoUnsignedWrap(), - hasNoSignedWrap()); - } - return State.get(this, 0); - } - case VPInstruction::CanonicalIVIncrementForPart: { auto *IV = State.get(getOperand(0), VPIteration(0, 0)); if (Part == 0) @@ -468,9 +462,6 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent, case VPInstruction::FirstOrderRecurrenceSplice: O << "first-order splice"; break; - case VPInstruction::CanonicalIVIncrement: - O << "VF * UF +"; - break; case VPInstruction::BranchOnCond: O << "branch-on-cond"; break; @@ -802,6 +793,7 @@ void VPWidenCastRecipe::print(raw_ostream &O, const Twine &Indent, O << Indent << "WIDEN-CAST "; printAsOperand(O, SlotTracker); O << " = " << Instruction::getOpcodeName(Opcode) << " "; + printFlags(O); printOperands(O, SlotTracker); O << " to " << *getResultType(); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index e8a6da8c32054a1262957b454d3acc69ec60f5b3..3bf91115debb7d5705e916f2f3fc2758fe76d1ee 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -22,11 +22,9 @@ class InductionDescriptor; class Instruction; class PHINode; class ScalarEvolution; -class Loop; class PredicatedScalarEvolution; class TargetLibraryInfo; class VPBuilder; -class VPRecipeBuilder; struct VPlanTransforms { /// Replaces the VPInstructions in \p Plan with corresponding diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index ac2883b30dc8ceebab30c848e314b2890350d79a..116acad8e8f3c69ec683c6c5b844e27b42b71c12 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -121,18 +121,11 @@ public: /// Remove a single \p User from the list of users. void removeUser(VPUser &User) { - bool Found = false; // The same user can be added multiple times, e.g. because the same VPValue // is used twice by the same VPUser. Remove a single one. - erase_if(Users, [&User, &Found](VPUser *Other) { - if (Found) - return false; - if (Other == &User) { - Found = true; - return true; - } - return false; - }); + auto *I = find(Users, &User); + if (I != Users.end()) + Users.erase(I); } typedef SmallVectorImpl::iterator user_iterator; @@ -303,6 +296,14 @@ public: "Op must be an operand of the recipe"); return false; } + + /// Returns true if the VPUser only uses the first part of operand \p Op. + /// Conservatively returns false. + virtual bool onlyFirstPartUsed(const VPValue *Op) const { + assert(is_contained(operands(), Op) && + "Op must be an operand of the recipe"); + return false; + } }; /// This class augments a recipe with a set of VPValues defined by the recipe. diff --git a/llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types_opaque_ptr.ll b/llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types_opaque_ptr.ll deleted file mode 100644 index e424683bbef341684d97091262a77f40c0d870bd..0000000000000000000000000000000000000000 --- a/llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types_opaque_ptr.ll +++ /dev/null @@ -1,179 +0,0 @@ -; RUN: opt -S -disable-output -passes='print' < %s 2>&1 | FileCheck %s - -; In the function below some of the accesses are done as float types and some -; are done as i32 types. When doing dependence analysis the type should not -; matter if it can be determined that they are the same size. - -%int_pair = type { i32, i32 } - -; CHECK-LABEL: function 'backdep_type_size_equivalence': -; CHECK-NEXT: loop: -; CHECK-NEXT: Memory dependences are safe with a maximum safe vector width of 3200 bits -; CHECK-NEXT: Dependences: -; CHECK-NEXT: Forward: -; CHECK-NEXT: %ld.f32 = load float, ptr %gep.iv, align 8 -> -; CHECK-NEXT: store i32 %indvars.iv.i32, ptr %gep.iv, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: Forward: -; CHECK-NEXT: %ld.f32 = load float, ptr %gep.iv, align 8 -> -; CHECK-NEXT: store float %val, ptr %gep.iv.min.100, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: BackwardVectorizable: -; CHECK-NEXT: store float %val, ptr %gep.iv.min.100, align 8 -> -; CHECK-NEXT: store i32 %indvars.iv.i32, ptr %gep.iv, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: Run-time memory checks: -; CHECK-NEXT: Grouped accesses: - -define void @backdep_type_size_equivalence(ptr nocapture %vec, i64 %n) { -entry: - br label %loop - -loop: - %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %loop ] - - ;; Load from vec[indvars.iv].x as float - %gep.iv = getelementptr inbounds %int_pair, ptr %vec, i64 %indvars.iv, i32 0 - %ld.f32 = load float, ptr %gep.iv, align 8 - %val = fmul fast float %ld.f32, 5.0 - - ;; Store to vec[indvars.iv - 100].x as float - %indvars.iv.min.100 = add nsw i64 %indvars.iv, -100 - %gep.iv.min.100 = getelementptr inbounds %int_pair, ptr %vec, i64 %indvars.iv.min.100, i32 0 - store float %val, ptr %gep.iv.min.100, align 8 - - ;; Store to vec[indvars.iv].x as i32, creating a backward dependency between - ;; the two stores with different element types but the same element size. - %indvars.iv.i32 = trunc i64 %indvars.iv to i32 - store i32 %indvars.iv.i32, ptr %gep.iv, align 8 - - ;; Store to vec[indvars.iv].y as i32, strided accesses should be independent - ;; between the two stores with different element types but the same element size. - %gep.iv.1 = getelementptr inbounds %int_pair, ptr %vec, i64 %indvars.iv, i32 1 - store i32 %indvars.iv.i32, ptr %gep.iv.1, align 8 - - ;; Store to vec[indvars.iv + n].y as i32, to verify no dependence in the case - ;; of unknown dependence distance. - %indvars.iv.n = add nuw nsw i64 %indvars.iv, %n - %gep.iv.n = getelementptr inbounds %int_pair, ptr %vec, i64 %indvars.iv.n, i32 1 - store i32 %indvars.iv.i32, ptr %gep.iv.n, align 8 - - ;; Loop condition. - %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 - %cond = icmp eq i64 %indvars.iv.next, %n - br i1 %cond, label %exit, label %loop - -exit: - ret void -} - -; In the function below one of the accesses is done as i19 type, which has a -; different store size than the i32 type, even though their alloc sizes are -; equivalent. This is a negative test to ensure that they are not analyzed as -; in the tests above. -; -; CHECK-LABEL: function 'backdep_type_store_size_equivalence': -; CHECK-NEXT: loop: -; CHECK-NEXT: Report: unsafe dependent memory operations in loop. -; CHECK-NEXT: Unknown data dependence. -; CHECK-NEXT: Dependences: -; CHECK-NEXT: Unknown: -; CHECK-NEXT: %ld.f32 = load float, ptr %gep.iv, align 8 -> -; CHECK-NEXT: store i19 %indvars.iv.i19, ptr %gep.iv, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: Run-time memory checks: -; CHECK-NEXT: Grouped accesses: - -define void @backdep_type_store_size_equivalence(ptr nocapture %vec, i64 %n) { -entry: - br label %loop - -loop: - %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %loop ] - - ;; Load from vec[indvars.iv].x as float - %gep.iv = getelementptr inbounds %int_pair, ptr %vec, i64 %indvars.iv, i32 0 - %ld.f32 = load float, ptr %gep.iv, align 8 - %val = fmul fast float %ld.f32, 5.0 - - ;; Store to vec[indvars.iv].x as i19. - %indvars.iv.i19 = trunc i64 %indvars.iv to i19 - store i19 %indvars.iv.i19, ptr %gep.iv, align 8 - - ;; Loop condition. - %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 - %cond = icmp eq i64 %indvars.iv.next, %n - br i1 %cond, label %exit, label %loop - -exit: - ret void -} - -; In the function below some of the accesses are done as double types and some -; are done as i64 and i32 types. This is a negative test to ensure that they -; are not analyzed as in the tests above. - -; CHECK-LABEL: function 'neg_dist_dep_type_size_equivalence': -; CHECK-NEXT: loop: -; CHECK-NEXT: Report: unsafe dependent memory operations in loop. -; CHECK-NEXT: Unknown data dependence. -; CHECK-NEXT: Dependences: -; CHECK-NEXT: Unknown: -; CHECK-NEXT: %ld.f64 = load double, ptr %gep.iv, align 8 -> -; CHECK-NEXT: store i32 %ld.i64.i32, ptr %gep.iv.n, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: Unknown: -; CHECK-NEXT: %ld.i64 = load i64, ptr %gep.iv, align 8 -> -; CHECK-NEXT: store i32 %ld.i64.i32, ptr %gep.iv.n, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: BackwardVectorizableButPreventsForwarding: -; CHECK-NEXT: %ld.f64 = load double, ptr %gep.iv, align 8 -> -; CHECK-NEXT: store double %val, ptr %gep.iv.101, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: ForwardButPreventsForwarding: -; CHECK-NEXT: store double %val, ptr %gep.iv.101, align 8 -> -; CHECK-NEXT: %ld.i64 = load i64, ptr %gep.iv, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: Unknown: -; CHECK-NEXT: store double %val, ptr %gep.iv.101, align 8 -> -; CHECK-NEXT: store i32 %ld.i64.i32, ptr %gep.iv.n, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: Run-time memory checks: -; CHECK-NEXT: Grouped accesses: - -define void @neg_dist_dep_type_size_equivalence(ptr nocapture %vec, i64 %n) { -entry: - br label %loop - -loop: - %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %loop ] - - ;; Load from vec[indvars.iv] as double - %gep.iv = getelementptr i64, ptr %vec, i64 %indvars.iv - %ld.f64 = load double, ptr %gep.iv, align 8 - %val = fmul fast double %ld.f64, 5.0 - - ;; Store to vec[indvars.iv + 101] as double - %indvars.iv.101 = add nsw i64 %indvars.iv, 101 - %gep.iv.101 = getelementptr i64, ptr %vec, i64 %indvars.iv.101 - store double %val, ptr %gep.iv.101, align 8 - - ;; Read from vec[indvars.iv] as i64 creating - ;; a forward but prevents forwarding dependence - ;; with different types but same sizes. - %ld.i64 = load i64, ptr %gep.iv, align 8 - - ;; Different sizes - %indvars.iv.n = add nuw nsw i64 %indvars.iv, %n - %gep.iv.n = getelementptr inbounds i64, ptr %vec, i64 %indvars.iv.n - %ld.i64.i32 = trunc i64 %ld.i64 to i32 - store i32 %ld.i64.i32, ptr %gep.iv.n, align 8 - - ;; Loop condition. - %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 - %cond = icmp eq i64 %indvars.iv.next, %n - br i1 %cond, label %exit, label %loop - -exit: - ret void -} diff --git a/llvm/test/Analysis/LoopAccessAnalysis/num-iters-for-store-load-conflict.ll b/llvm/test/Analysis/LoopAccessAnalysis/num-iters-for-store-load-conflict.ll new file mode 100644 index 0000000000000000000000000000000000000000..d3eda21dee27e6e12a3c380991d1bf6931acd1c3 --- /dev/null +++ b/llvm/test/Analysis/LoopAccessAnalysis/num-iters-for-store-load-conflict.ll @@ -0,0 +1,310 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes='print' -disable-output < %s 2>&1 | FileCheck %s + +define void @forward_dist_7(ptr %A, ptr noalias %B) { +; CHECK-LABEL: 'forward_dist_7' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: ForwardButPreventsForwarding: +; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 -> +; CHECK-NEXT: %l = load i32, ptr %gep.1, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %next, %loop ] + %gep.1 = getelementptr i32, ptr %A, i64 %iv + %gep.2 = getelementptr i32, ptr %gep.1, i64 7 + store i32 0, ptr %gep.2, align 4 + %l = load i32, ptr %gep.1 + store i32 %l, ptr %B + %next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv, 1000 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @forward_dist_9(ptr %A, ptr noalias %B) { +; CHECK-LABEL: 'forward_dist_9' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: ForwardButPreventsForwarding: +; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 -> +; CHECK-NEXT: %l = load i32, ptr %gep.1, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %next, %loop ] + %gep.1 = getelementptr i32, ptr %A, i64 %iv + %gep.2 = getelementptr i32, ptr %gep.1, i64 9 + store i32 0, ptr %gep.2, align 4 + %l = load i32, ptr %gep.1 + store i32 %l, ptr %B + %next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv, 1000 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @forward_dist_11(ptr %A, ptr noalias %B) { +; CHECK-LABEL: 'forward_dist_11' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: ForwardButPreventsForwarding: +; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 -> +; CHECK-NEXT: %l = load i32, ptr %gep.1, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %next, %loop ] + %gep.1 = getelementptr i32, ptr %A, i64 %iv + %gep.2 = getelementptr i32, ptr %gep.1, i64 9 + store i32 0, ptr %gep.2, align 4 + %l = load i32, ptr %gep.1 + store i32 %l, ptr %B + %next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv, 1000 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @forward_dist_13(ptr %A, ptr noalias %B) { +; CHECK-LABEL: 'forward_dist_13' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: ForwardButPreventsForwarding: +; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 -> +; CHECK-NEXT: %l = load i32, ptr %gep.1, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %next, %loop ] + %gep.1 = getelementptr i32, ptr %A, i64 %iv + %gep.2 = getelementptr i32, ptr %gep.1, i64 13 + store i32 0, ptr %gep.2, align 4 + %l = load i32, ptr %gep.1 + store i32 %l, ptr %B + %next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv, 1000 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @forward_dist_15(ptr %A, ptr noalias %B) { +; CHECK-LABEL: 'forward_dist_15' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: ForwardButPreventsForwarding: +; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 -> +; CHECK-NEXT: %l = load i32, ptr %gep.1, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %next, %loop ] + %gep.1 = getelementptr i32, ptr %A, i64 %iv + %gep.2 = getelementptr i32, ptr %gep.1, i64 13 + store i32 0, ptr %gep.2, align 4 + %l = load i32, ptr %gep.1 + store i32 %l, ptr %B + %next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv, 1000 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @forward_dist_17(ptr %A, ptr noalias %B) { +; CHECK-LABEL: 'forward_dist_17' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: ForwardButPreventsForwarding: +; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 -> +; CHECK-NEXT: %l = load i32, ptr %gep.1, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %next, %loop ] + %gep.1 = getelementptr i32, ptr %A, i64 %iv + %gep.2 = getelementptr i32, ptr %gep.1, i64 17 + store i32 0, ptr %gep.2, align 4 + %l = load i32, ptr %gep.1 + store i32 %l, ptr %B + %next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv, 1000 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @forward_dist_19(ptr %A, ptr noalias %B) { +; CHECK-LABEL: 'forward_dist_19' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: ForwardButPreventsForwarding: +; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 -> +; CHECK-NEXT: %l = load i32, ptr %gep.1, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %next, %loop ] + %gep.1 = getelementptr i32, ptr %A, i64 %iv + %gep.2 = getelementptr i32, ptr %gep.1, i64 19 + store i32 0, ptr %gep.2, align 4 + %l = load i32, ptr %gep.1 + store i32 %l, ptr %B + %next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv, 1000 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +@A = global [37 x [37 x double]] zeroinitializer, align 8 + +define void @unknown_loop_bounds(i64 %x, i64 %y) { +; CHECK-LABEL: 'unknown_loop_bounds' +; CHECK-NEXT: inner: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Backward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: BackwardVectorizableButPreventsForwarding: +; CHECK-NEXT: %l = load double, ptr %gep.0, align 8 -> +; CHECK-NEXT: store double %l, ptr %gep.1, align 8 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; CHECK-NEXT: outer.header: +; CHECK-NEXT: Report: loop is not the innermost loop +; CHECK-NEXT: Dependences: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %outer.header + +outer.header: + %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ] + %outer.iv.next = add nuw nsw i64 %outer.iv, 1 + br label %inner + +inner: + %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ] + %gep.0 = getelementptr inbounds [37 x [37 x double]], ptr @A, i64 0, i64 %outer.iv, i64 %inner.iv + %l = load double, ptr %gep.0, align 8 + %gep.1 = getelementptr inbounds [37 x [37 x double]], ptr @A, i64 0, i64 %outer.iv.next, i64 %inner.iv + store double %l, ptr %gep.1, align 8 + %inner.iv.next = add nuw nsw i64 %inner.iv, 1 + %inner.ec = icmp eq i64 %inner.iv.next, %y + br i1 %inner.ec, label %outer.latch, label %inner + +outer.latch: + %outer.ec = icmp eq i64 %outer.iv.next, %x + br i1 %outer.ec, label %exit, label %outer.header + +exit: + ret void +} diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count.ll b/llvm/test/Analysis/ScalarEvolution/trip-count.ll index 5973d52d05ea6f8b233b155e4f73c4f6382ebcda..22e49ebdbf4dbfd24168dc3a1437038d7b508dfe 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count.ll @@ -123,3 +123,28 @@ loop: leave: ret void } + +define void @non_zero_from_loop_guard(i16 %n) { +; CHECK-LABEL: 'non_zero_from_loop_guard' +; CHECK-NEXT: Determining loop execution counts for: @non_zero_from_loop_guard +; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (1 umax (%n /u 2))) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 32766 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (1 umax (%n /u 2))) +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (1 umax (%n /u 2))) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; +entry: + %shr = lshr i16 %n, 1 + %precond = icmp ult i16 %n, 2 + br i1 %precond, label %exit, label %loop + +loop: + %iv = phi i16 [ %inc, %loop ], [ 0, %entry ] + %inc = add nuw nsw i16 %iv, 1 + %cmp = icmp ult i16 %inc, %shr + br i1 %cmp, label %loop, label %exit + +exit: + ret void +} diff --git a/llvm/test/Analysis/ValueTracking/knownbits-and-or-xor-lowbit.ll b/llvm/test/Analysis/ValueTracking/knownbits-and-or-xor-lowbit.ll index f44948e6a089f88236969aa695067a23859160bd..4ca7ed9eda7bbc7db1d69fbbef132f7944e4c7c2 100644 --- a/llvm/test/Analysis/ValueTracking/knownbits-and-or-xor-lowbit.ll +++ b/llvm/test/Analysis/ValueTracking/knownbits-and-or-xor-lowbit.ll @@ -147,10 +147,9 @@ define i1 @sub_YX_and_bit0_is_zero_fail(i8 %x, i8 %C) nounwind { define <2 x i1> @sub_YX_xor_bit0_is_one_fail(<2 x i8> %x, <2 x i8> %C) nounwind { ; CHECK-LABEL: @sub_YX_xor_bit0_is_one_fail( -; CHECK-NEXT: [[TMP1:%.*]] = xor <2 x i8> [[X:%.*]], -; CHECK-NEXT: [[Y:%.*]] = add <2 x i8> [[TMP1]], [[C:%.*]] -; CHECK-NEXT: [[W:%.*]] = xor <2 x i8> [[Y]], [[X]] -; CHECK-NEXT: [[R:%.*]] = icmp eq <2 x i8> [[W]], +; CHECK-NEXT: [[TMP1:%.*]] = sub <2 x i8> [[X:%.*]], [[C:%.*]] +; CHECK-NEXT: [[TMP2:%.*]] = xor <2 x i8> [[TMP1]], [[X]] +; CHECK-NEXT: [[R:%.*]] = icmp eq <2 x i8> [[TMP2]], ; CHECK-NEXT: ret <2 x i1> [[R]] ; %C1 = sub <2 x i8> %C, diff --git a/llvm/test/Assembler/constant-splat-diagnostics.ll b/llvm/test/Assembler/constant-splat-diagnostics.ll new file mode 100644 index 0000000000000000000000000000000000000000..464250d03ea283cc22a5d415f68074bd56486a45 --- /dev/null +++ b/llvm/test/Assembler/constant-splat-diagnostics.ll @@ -0,0 +1,40 @@ +; RUN: rm -rf %t && split-file %s %t + +; RUN: not llvm-as < %t/not_a_constant.ll -o /dev/null 2>&1 | FileCheck -check-prefix=NOT_A_CONSTANT %s +; RUN: not llvm-as < %t/not_a_sclar.ll -o /dev/null 2>&1 | FileCheck -check-prefix=NOT_A_SCALAR %s +; RUN: not llvm-as < %t/not_a_vector.ll -o /dev/null 2>&1 | FileCheck -check-prefix=NOT_A_VECTOR %s +; RUN: not llvm-as < %t/wrong_explicit_type.ll -o /dev/null 2>&1 | FileCheck -check-prefix=WRONG_EXPLICIT_TYPE %s +; RUN: not llvm-as < %t/wrong_implicit_type.ll -o /dev/null 2>&1 | FileCheck -check-prefix=WRONG_IMPLICIT_TYPE %s + +;--- not_a_constant.ll +; NOT_A_CONSTANT: error: expected instruction opcode +define <4 x i32> @not_a_constant(i32 %a) { + %splat = splat (i32 %a) + ret %splat +} + +;--- not_a_sclar.ll +; NOT_A_SCALAR: error: constant expression type mismatch: got type '<1 x i32>' but expected 'i32' +define <4 x i32> @not_a_scalar() { + ret <4 x i32> splat (<1 x i32> ) +} + +;--- not_a_vector.ll +; NOT_A_VECTOR: error: vector constant must have vector type +define <4 x i32> @not_a_vector() { + ret i32 splat (i32 7) +} + +;--- wrong_explicit_type.ll +; WRONG_EXPLICIT_TYPE: error: constant expression type mismatch: got type 'i8' but expected 'i32' +define <4 x i32> @wrong_explicit_type() { + ret <4 x i32> splat (i8 7) +} + +;--- wrong_implicit_type.ll +; WRONG_IMPLICIT_TYPE: error: constant expression type mismatch: got type 'i8' but expected 'i32' +define void @wrong_implicit_type(<4 x i32> %a) { + %add = add <4 x i32> %a, splat (i8 7) + ret void +} + diff --git a/llvm/test/Assembler/constant-splat.ll b/llvm/test/Assembler/constant-splat.ll new file mode 100644 index 0000000000000000000000000000000000000000..f3ec0c8340aa15714c377f80bc28996e79e0c21d --- /dev/null +++ b/llvm/test/Assembler/constant-splat.ll @@ -0,0 +1,67 @@ +; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s + +; NOTE: Tests the expansion of the "splat" shorthand method to create vector +; constants. Future work will change how "splat" is expanded, ultimately +; leading to a point where "splat" is emitted as the disassembly. + +@my_global = external global i32 + +; CHECK: @constant.splat.i1 = constant <1 x i1> +@constant.splat.i1 = constant <1 x i1> splat (i1 true) + +; CHECK: @constant.splat.i32 = constant <5 x i32> +@constant.splat.i32 = constant <5 x i32> splat (i32 7) + +; CHECK: @constant.splat.i128 = constant <2 x i128> +@constant.splat.i128 = constant <2 x i128> splat (i128 85070591730234615870450834276742070272) + +; CHECK: @constant.splat.f16 = constant <4 x half> +@constant.splat.f16 = constant <4 x half> splat (half 0xHBC00) + +; CHECK: @constant.splat.f32 = constant <5 x float> +@constant.splat.f32 = constant <5 x float> splat (float -2.000000e+00) + +; CHECK: @constant.splat.f64 = constant <3 x double> +@constant.splat.f64 = constant <3 x double> splat (double -3.000000e+00) + +; CHECK: @constant.splat.128 = constant <2 x fp128> +@constant.splat.128 = constant <2 x fp128> splat (fp128 0xL00000000000000018000000000000000) + +; CHECK: @constant.splat.bf16 = constant <4 x bfloat> +@constant.splat.bf16 = constant <4 x bfloat> splat (bfloat 0xRC0A0) + +; CHECK: @constant.splat.x86_fp80 = constant <3 x x86_fp80> +@constant.splat.x86_fp80 = constant <3 x x86_fp80> splat (x86_fp80 0xK4000C8F5C28F5C28F800) + +; CHECK: @constant.splat.ppc_fp128 = constant <1 x ppc_fp128> +@constant.splat.ppc_fp128 = constant <1 x ppc_fp128> splat (ppc_fp128 0xM80000000000000000000000000000000) + +; CHECK: @constant.splat.global.ptr = constant <4 x ptr> +@constant.splat.global.ptr = constant <4 x ptr> splat (ptr @my_global) + +define void @add_fixed_lenth_vector_splat_i32(<4 x i32> %a) { +; CHECK: %add = add <4 x i32> %a, + %add = add <4 x i32> %a, splat (i32 137) + ret void +} + +define <4 x i32> @ret_fixed_lenth_vector_splat_i32() { +; CHECK: ret <4 x i32> + ret <4 x i32> splat (i32 56) +} + +define void @add_fixed_lenth_vector_splat_double( %a) { +; CHECK: %add = fadd %a, shufflevector ( insertelement ( poison, double 5.700000e+00, i64 0), poison, zeroinitializer) + %add = fadd %a, splat (double 5.700000e+00) + ret void +} + +define @ret_scalable_vector_splat_i32() { +; CHECK: ret shufflevector ( insertelement ( poison, i32 78, i64 0), poison, zeroinitializer) + ret splat (i32 78) +} + +define @ret_scalable_vector_ptr() { +; CHECK: ret shufflevector ( insertelement ( poison, ptr @my_global, i64 0), poison, zeroinitializer) + ret splat (ptr @my_global) +} diff --git a/llvm/test/Assembler/named-metadata.ll b/llvm/test/Assembler/named-metadata.ll index 9fa37a7989d4f8757f58ae395fc1fee766bc0494..32ebf2bfd26f37f9df5b20a94c8b22939108fa95 100644 --- a/llvm/test/Assembler/named-metadata.ll +++ b/llvm/test/Assembler/named-metadata.ll @@ -23,3 +23,7 @@ ; when emitting it, followed by xfoo. !\xfoo = !{!0, !1, !2} ; CHECK: !\5Cxfoo = !{!0, !1, !2} + +; Make sure we handle escapes with the high bit set correctly. +!\FFfoo = !{!0, !1, !2} +; CHECK: !\FFfoo = !{!0, !1, !2} diff --git a/llvm/test/Assembler/struct-ret-without-upgrade.ll b/llvm/test/Assembler/struct-ret-without-upgrade.ll index 992b2f9f767fa67419fc4a8fe6d9b58ee9d42628..14f931c23abb6b266a1caa37ec9d4ff8599f9024 100644 --- a/llvm/test/Assembler/struct-ret-without-upgrade.ll +++ b/llvm/test/Assembler/struct-ret-without-upgrade.ll @@ -15,4 +15,15 @@ define %ty @test(%ty %arg) { ret %ty %copy } +define %ty @test_not_real_intrinsic() { +; CHECK-LABEL: @test_not_real_intrinsic( +; CHECK-NEXT: [[RET:%.*]] = call [[TY:%.*]] @llvm.dummy() +; CHECK-NEXT: ret [[TY]] [[RET]] +; + %ret = call %ty @llvm.dummy() + ret %ty %ret +} + +declare %ty @llvm.dummy() + declare %ty @llvm.ssa.copy.s_tys(%ty) diff --git a/llvm/test/Assembler/summary-parsing-error.ll b/llvm/test/Assembler/summary-parsing-error.ll new file mode 100644 index 0000000000000000000000000000000000000000..47e5c64e129841a344d98bfffbcee8c3f091a14e --- /dev/null +++ b/llvm/test/Assembler/summary-parsing-error.ll @@ -0,0 +1,4 @@ +; RUN: not llvm-as < %s 2>&1 | FileCheck %s + +; CHECK: Reference to undefined global "does_not_exist" +^0 = gv: (name: "does_not_exist") diff --git a/llvm/test/Bindings/llvm-c/echo.ll b/llvm/test/Bindings/llvm-c/echo.ll index 0775cbb673e4ee9390e3c6e9ed1d676dc3458f80..2e195beebd7bb3dc16e185e44f11337de522c5b1 100644 --- a/llvm/test/Bindings/llvm-c/echo.ll +++ b/llvm/test/Bindings/llvm-c/echo.ll @@ -269,6 +269,17 @@ exit: ret void } +define void @operandbundles() personality ptr @personalityFn { + call void @decl() [ "foo"(), "bar\00x"(i32 0, ptr null, token none) ] + invoke void @decl() [ "baz"(label %bar) ] to label %foo unwind label %bar +foo: + ret void +bar: + %1 = landingpad { ptr, i32 } + cleanup + ret void +} + define void @with_debuginfo() !dbg !4 { ret void, !dbg !7 } diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll index 575cd6b874e3563152a996d3a11e6f09bd57d1b5..92ddc6309546f3a5bcc083cf03aec43bd84823db 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll @@ -1,5 +1,5 @@ -; RUN: llc -O0 -aarch64-enable-atomic-cfg-tidy=0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s -; RUN: llc -O3 -aarch64-enable-atomic-cfg-tidy=0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefix=O3 +; RUN: llc -O0 -aarch64-enable-atomic-cfg-tidy=0 -mattr=+lse -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s +; RUN: llc -O3 -aarch64-enable-atomic-cfg-tidy=0 -mattr=+lse -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefix=O3 ; This file checks that the translation from llvm IR to generic MachineInstr ; is correct. @@ -2077,190 +2077,147 @@ done: } ; Try a monotonic atomicrmw xchg -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_xchg(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_xchg ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_XCHG [[ADDR]](p0), [[VAL]] :: (load store monotonic (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw xchg ptr %addr, i256 1 monotonic - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_XCHG [[ADDR]](p0), [[VAL]] :: (load store monotonic (s32) on %ir.addr) + %oldval = atomicrmw xchg ptr %addr, i32 1 monotonic + ret i32 %oldval } ; Try an acquire atomicrmw add -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_add(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_add ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_ADD [[ADDR]](p0), [[VAL]] :: (load store acquire (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw add ptr %addr, i256 1 acquire - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[ADDR]](p0), [[VAL]] :: (load store acquire (s32) on %ir.addr) + %oldval = atomicrmw add ptr %addr, i32 1 acquire + ret i32 %oldval } ; Try a release atomicrmw sub -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_sub(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_sub ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_SUB [[ADDR]](p0), [[VAL]] :: (load store release (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw sub ptr %addr, i256 1 release - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_SUB [[ADDR]](p0), [[VAL]] :: (load store release (s32) on %ir.addr) + %oldval = atomicrmw sub ptr %addr, i32 1 release + ret i32 %oldval } ; Try an acq_rel atomicrmw and -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_and(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_and ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_AND [[ADDR]](p0), [[VAL]] :: (load store acq_rel (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw and ptr %addr, i256 1 acq_rel - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc -} - -; Try an seq_cst atomicrmw nand -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_AND [[ADDR]](p0), [[VAL]] :: (load store acq_rel (s32) on %ir.addr) + %oldval = atomicrmw and ptr %addr, i32 1 acq_rel + ret i32 %oldval +} + +; Try an seq_cst atomicrmw nand. NAND isn't supported by LSE, so it +; expands to G_ATOMIC_CMPXCHG_WITH_SUCCESS. define i32 @test_atomicrmw_nand(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_nand ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): +; CHECK-NEXT: successors: %bb.2(0x80000000) ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_NAND [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw nand ptr %addr, i256 1 seq_cst - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[NEG1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1 +; CHECK-NEXT: [[OLDVALSTART:%[0-9]+]]:_(s32) = G_LOAD [[ADDR]](p0) :: (load (s32) from %ir.addr) +; CHECK: bb.2.atomicrmw.start: +; CHECK-NEXT: successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}}) +; CHECK: [[OLDVAL:%[0-9]+]]:_(s32) = G_PHI [[OLDVALSTART]](s32), %bb.1, [[OLDVALRES:%[0-9]+]](s32), %bb.2 +; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OLDVAL]], [[VAL]] +; CHECK-NEXT: [[NEWVAL:%[0-9]+]]:_(s32) = G_XOR [[AND]], [[NEG1]] +; CHECK: [[OLDVALRES]]:_(s32), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store seq_cst seq_cst (s32) on %ir.addr) +; CHECK-NEXT: G_BRCOND [[SUCCESS]](s1), %bb.3 +; CHECK-NEXT: G_BR %bb.2 +; CHECK: bb.3.atomicrmw.end: + %oldval = atomicrmw nand ptr %addr, i32 1 seq_cst + ret i32 %oldval } ; Try an seq_cst atomicrmw or -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_or(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_or ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_OR [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw or ptr %addr, i256 1 seq_cst - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_OR [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s32) on %ir.addr) + %oldval = atomicrmw or ptr %addr, i32 1 seq_cst + ret i32 %oldval } ; Try an seq_cst atomicrmw xor -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_xor(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_xor ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_XOR [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw xor ptr %addr, i256 1 seq_cst - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_XOR [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s32) on %ir.addr) + %oldval = atomicrmw xor ptr %addr, i32 1 seq_cst + ret i32 %oldval } ; Try an seq_cst atomicrmw min -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_min(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_min ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_MIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw min ptr %addr, i256 1 seq_cst - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_MIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s32) on %ir.addr) + %oldval = atomicrmw min ptr %addr, i32 1 seq_cst + ret i32 %oldval } ; Try an seq_cst atomicrmw max -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_max(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_max ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_MAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw max ptr %addr, i256 1 seq_cst - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_MAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s32) on %ir.addr) + %oldval = atomicrmw max ptr %addr, i32 1 seq_cst + ret i32 %oldval } ; Try an seq_cst atomicrmw unsigned min -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_umin(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_umin ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_UMIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw umin ptr %addr, i256 1 seq_cst - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_UMIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s32) on %ir.addr) + %oldval = atomicrmw umin ptr %addr, i32 1 seq_cst + ret i32 %oldval } ; Try an seq_cst atomicrmw unsigned max -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_umax(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_umax ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_UMAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw umax ptr %addr, i256 1 seq_cst - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_UMAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s32) on %ir.addr) + %oldval = atomicrmw umax ptr %addr, i32 1 seq_cst + ret i32 %oldval } @addr = global ptr null diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index ae15e74a43277ab85f34c21cd2b4b01d7bcea33d..178db852e35b7e121bdc64eb686d381848f554a6 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -258,6 +258,9 @@ # DEBUG-NEXT: G_FENCE (opcode {{[0-9]+}}): 0 type indices # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: G_PREFETCH (opcode {{[0-9]+}}): 1 type index, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected +# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: G_BRCOND (opcode {{[0-9]+}}): 1 type index, 0 imm indices # DEBUG-NEXT: .. the first uncovered type index: 1, OK # DEBUG-NEXT: .. the first uncovered imm index: 0, OK diff --git a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll index 55750ab34e17a03ccdaaea1907c31c470c57f657..3c8aca5145261d0716c4689560dacf0605c1eddd 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll @@ -69,11 +69,10 @@ define void @insert_vec_v23i32_uaddlv_from_v8i16(ptr %0) { ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: movi.2d v0, #0000000000000000 ; CHECK-NEXT: movi.2d v2, #0000000000000000 -; CHECK-NEXT: add x8, x0, #88 +; CHECK-NEXT: str wzr, [x0, #88] ; CHECK-NEXT: uaddlv.8h s1, v0 ; CHECK-NEXT: stp q0, q0, [x0, #16] ; CHECK-NEXT: stp q0, q0, [x0, #48] -; CHECK-NEXT: st1.s { v0 }[2], [x8] ; CHECK-NEXT: str d0, [x0, #80] ; CHECK-NEXT: mov.s v2[0], v1[0] ; CHECK-NEXT: ucvtf.4s v1, v2 diff --git a/llvm/test/CodeGen/AArch64/arm64-promote-const-complex-initializers.ll b/llvm/test/CodeGen/AArch64/arm64-promote-const-complex-initializers.ll index 86ebf803c578319942b7b1e2108bdd6030ad7865..bbd2acbab4246a333967f17a55a0e5a4fc2676f1 100644 --- a/llvm/test/CodeGen/AArch64/arm64-promote-const-complex-initializers.ll +++ b/llvm/test/CodeGen/AArch64/arm64-promote-const-complex-initializers.ll @@ -30,10 +30,10 @@ define [1 x <4 x float>] @test1() { define [1 x <4 x float>] @test2() { ; CHECK-LABEL: .p2align 4, 0x0 ; -- Begin function test2 ; CHECK-NEXT: lCPI1_0: -; CHECK-NEXT: .long 0x00000000 ; float 0 -; CHECK-NEXT: .long 0x00000000 ; float 0 -; CHECK-NEXT: .long 0x00000000 ; float 0 -; CHECK-NEXT: .long 0x3f800000 ; float 1 +; CHECK-NEXT: .long 0x80000000 ; float -0 +; CHECK-NEXT: .long 0x80000000 ; float -0 +; CHECK-NEXT: .long 0x80000000 ; float -0 +; CHECK-NEXT: .long 0xbf800000 ; float -1 ; CHECK-NEXT: .section __TEXT,__text,regular,pure_instructions ; CHECK-NEXT: .globl _test2 ; CHECK-NEXT: .p2align 2 @@ -43,17 +43,7 @@ define [1 x <4 x float>] @test2() { ; CHECK-NEXT: Lloh2: ; CHECK-NEXT: adrp x8, lCPI1_0@PAGE ; CHECK-NEXT: Lloh3: -; CHECK-NEXT: ldr q1, [x8, lCPI1_0@PAGEOFF] -; CHECK-NEXT: mov s2, v1[1] -; CHECK-NEXT: fneg s0, s1 -; CHECK-NEXT: mov s3, v1[2] -; CHECK-NEXT: mov s1, v1[3] -; CHECK-NEXT: fneg s2, s2 -; CHECK-NEXT: fneg s3, s3 -; CHECK-NEXT: fneg s1, s1 -; CHECK-NEXT: mov.s v0[1], v2[0] -; CHECK-NEXT: mov.s v0[2], v3[0] -; CHECK-NEXT: mov.s v0[3], v1[0] +; CHECK-NEXT: ldr q0, [x8, lCPI1_0@PAGEOFF] ; CHECK-NEXT: ret ; %constexpr = fneg float extractelement (<4 x float> bitcast (<1 x i128> to <4 x float>), i32 0) diff --git a/llvm/test/CodeGen/AArch64/atomic-oversize.ll b/llvm/test/CodeGen/AArch64/atomic-oversize.ll new file mode 100644 index 0000000000000000000000000000000000000000..9065311a9aaa14741fa47d71a237f35a69194e2d --- /dev/null +++ b/llvm/test/CodeGen/AArch64/atomic-oversize.ll @@ -0,0 +1,11 @@ +; RUN: llc -march=aarch64 < %s | FileCheck %s + +; Atomics larger than 128-bit are unsupported, and emit libcalls. +define void @test(ptr %a) nounwind { +; CHECK-LABEL: test: +; CHECK: bl __atomic_load +; CHECK: bl __atomic_store + %1 = load atomic i256, ptr %a seq_cst, align 32 + store atomic i256 %1, ptr %a seq_cst, align 32 + ret void +} diff --git a/llvm/test/CodeGen/AArch64/cmp-select-sign.ll b/llvm/test/CodeGen/AArch64/cmp-select-sign.ll index d16b5786a996557846cb509c5d2ed4ee20409329..09a6e26fe5a403a8cdf13189262e1f848a91033c 100644 --- a/llvm/test/CodeGen/AArch64/cmp-select-sign.ll +++ b/llvm/test/CodeGen/AArch64/cmp-select-sign.ll @@ -176,12 +176,11 @@ define <4 x i32> @sign_4xi32_multi_use(<4 x i32> %a) { ; CHECK-NEXT: str x30, [sp, #16] // 8-byte Folded Spill ; CHECK-NEXT: .cfi_def_cfa_offset 32 ; CHECK-NEXT: .cfi_offset w30, -16 -; CHECK-NEXT: movi v1.2d, #0xffffffffffffffff -; CHECK-NEXT: cmlt v2.4s, v0.4s, #0 -; CHECK-NEXT: orr v2.4s, #1 -; CHECK-NEXT: cmgt v1.4s, v0.4s, v1.4s -; CHECK-NEXT: str q2, [sp] // 16-byte Folded Spill -; CHECK-NEXT: xtn v0.4h, v1.4s +; CHECK-NEXT: cmlt v1.4s, v0.4s, #0 +; CHECK-NEXT: cmge v0.4s, v0.4s, #0 +; CHECK-NEXT: xtn v0.4h, v0.4s +; CHECK-NEXT: orr v1.4s, #1 +; CHECK-NEXT: str q1, [sp] // 16-byte Folded Spill ; CHECK-NEXT: bl use_4xi1 ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: ldr x30, [sp, #16] // 8-byte Folded Reload @@ -227,9 +226,8 @@ define <4 x i32> @not_sign_4xi32_2(<4 x i32> %a) { define <4 x i32> @not_sign_4xi32_3(<4 x i32> %a) { ; CHECK-LABEL: not_sign_4xi32_3: ; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0xffffffffffffffff ; CHECK-NEXT: adrp x8, .LCPI18_0 -; CHECK-NEXT: cmgt v0.4s, v0.4s, v1.4s +; CHECK-NEXT: cmge v0.4s, v0.4s, #0 ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI18_0] ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b ; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s diff --git a/llvm/test/CodeGen/AArch64/dag-numsignbits.ll b/llvm/test/CodeGen/AArch64/dag-numsignbits.ll index 575102ce4ff962968b4a2a5236c8ab28dccd7851..11f19b8c56965bf411b448d3c748ad7809e90fc2 100644 --- a/llvm/test/CodeGen/AArch64/dag-numsignbits.ll +++ b/llvm/test/CodeGen/AArch64/dag-numsignbits.ll @@ -8,13 +8,12 @@ define void @signbits_vXi1(<4 x i16> %a1) { ; CHECK: // %bb.0: ; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: adrp x8, .LCPI0_0 -; CHECK-NEXT: movi v2.4h, #1 ; CHECK-NEXT: mov w1, wzr +; CHECK-NEXT: mov w2, wzr ; CHECK-NEXT: dup v0.4h, v0.h[0] ; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI0_0] -; CHECK-NEXT: mov w2, wzr ; CHECK-NEXT: add v0.4h, v0.4h, v1.4h -; CHECK-NEXT: cmgt v0.4h, v2.4h, v0.4h +; CHECK-NEXT: cmle v0.4h, v0.4h, #0 ; CHECK-NEXT: umov w0, v0.h[0] ; CHECK-NEXT: umov w3, v0.h[3] ; CHECK-NEXT: b foo diff --git a/llvm/test/CodeGen/AArch64/fpenv.ll b/llvm/test/CodeGen/AArch64/fpenv.ll index 14bf7888918e5be33241a4007f2d704643ab9450..3a307f7731037a8f8a5d37f6c6edeec2fa33856f 100644 --- a/llvm/test/CodeGen/AArch64/fpenv.ll +++ b/llvm/test/CodeGen/AArch64/fpenv.ll @@ -1,70 +1,65 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s -define void @func_02(i32 %rm) { +define void @func_set_rounding_dyn(i32 %rm) { +; CHECK-LABEL: func_set_rounding_dyn: +; CHECK: // %bb.0: +; CHECK-NEXT: sub w9, w0, #1 +; CHECK-NEXT: mrs x8, FPCR +; CHECK-NEXT: and w9, w9, #0x3 +; CHECK-NEXT: and x8, x8, #0xffffffffff3fffff +; CHECK-NEXT: lsl w9, w9, #22 +; CHECK-NEXT: orr x8, x8, x9 +; CHECK-NEXT: msr FPCR, x8 +; CHECK-NEXT: ret call void @llvm.set.rounding(i32 %rm) ret void } -; CHECK-LABEL: func_02: -; CHECK: sub w9, w0, #1 -; CHECK: mrs x8, FPCR -; CHECK: and w9, w9, #0x3 -; CHECK: and x8, x8, #0xffffffffff3fffff -; CHECK: lsl w9, w9, #22 -; CHECK: orr x8, x8, x9 -; CHECK: msr FPCR, x8 -; CHECK: ret - - -define void @func_03() { +define void @func_set_rounding_toward_zero() { +; CHECK-LABEL: func_set_rounding_toward_zero: +; CHECK: // %bb.0: +; CHECK-NEXT: mrs x8, FPCR +; CHECK-NEXT: orr x8, x8, #0xc00000 +; CHECK-NEXT: msr FPCR, x8 +; CHECK-NEXT: ret call void @llvm.set.rounding(i32 0) ret void } -; CHECK-LABEL: func_03 -; CHECK: mrs x8, FPCR -; CHECK: orr x8, x8, #0xc00000 -; CHECK: msr FPCR, x8 -; CHECK: ret - - -define void @func_04() { +define void @func_set_rounding_tonearest_even() { +; CHECK-LABEL: func_set_rounding_tonearest_even: +; CHECK: // %bb.0: +; CHECK-NEXT: mrs x8, FPCR +; CHECK-NEXT: and x8, x8, #0xffffffffff3fffff +; CHECK-NEXT: msr FPCR, x8 +; CHECK-NEXT: ret call void @llvm.set.rounding(i32 1) ret void } -; CHECK-LABEL: func_04 -; CHECK: mrs x8, FPCR -; CHECK: and x8, x8, #0xffffffffff3fffff -; CHECK: msr FPCR, x8 -; CHECK: ret - - -define void @func_05() { +define void @func_set_rounding_upward() { +; CHECK-LABEL: func_set_rounding_upward: +; CHECK: // %bb.0: +; CHECK-NEXT: mrs x8, FPCR +; CHECK-NEXT: and x8, x8, #0xffffffffff3fffff +; CHECK-NEXT: orr x8, x8, #0x400000 +; CHECK-NEXT: msr FPCR, x8 +; CHECK-NEXT: ret call void @llvm.set.rounding(i32 2) ret void } - -; CHECK-LABEL: func_05 -; CHECK: mrs x8, FPCR -; CHECK: and x8, x8, #0xffffffffff3fffff -; CHECK: orr x8, x8, #0x400000 -; CHECK: msr FPCR, x8 -; CHECK: ret - - -define void @func_06() { +define void @func_set_rounding_downward() { +; CHECK-LABEL: func_set_rounding_downward: +; CHECK: // %bb.0: +; CHECK-NEXT: mrs x8, FPCR +; CHECK-NEXT: and x8, x8, #0xffffffffff3fffff +; CHECK-NEXT: orr x8, x8, #0x800000 +; CHECK-NEXT: msr FPCR, x8 +; CHECK-NEXT: ret call void @llvm.set.rounding(i32 3) ret void } -; CHECK-LABEL: func_06 -; CHECK: mrs x8, FPCR -; CHECK: and x8, x8, #0xffffffffff3fffff -; CHECK: orr x8, x8, #0x800000 -; CHECK: msr FPCR, x8 -; CHECK: ret - - declare void @llvm.set.rounding(i32) diff --git a/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll b/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll index 6ee3cb4892852f9144767b59517fcd92b7c84a50..ee3e808f9f9218cdde2e1c1a0e5b6fe25bd63a92 100644 --- a/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll +++ b/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll @@ -12,7 +12,6 @@ ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-x1 | FileCheck %s ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=neoverse-e1 | FileCheck %s ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=neoverse-n1 | FileCheck %s -; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=neoverse-n2 | FileCheck %s ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=neoverse-v1 | FileCheck %s ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=neoverse-512tvb | FileCheck %s ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m3 | FileCheck %s diff --git a/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll b/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll index 0abdda5954f570f76b1438edd0047b175f3d2145..2d594947682b58d2f24ff9b66ee3f8643b20b2e9 100644 --- a/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll +++ b/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll @@ -7,7 +7,7 @@ define <8 x i8> @cmeq8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmeq v0.8b, v0.8b, v1.8b ; CHECK-NEXT: ret - %tmp3 = icmp eq <8 x i8> %A, %B; + %tmp3 = icmp eq <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -17,7 +17,7 @@ define <16 x i8> @cmeq16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmeq v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret - %tmp3 = icmp eq <16 x i8> %A, %B; + %tmp3 = icmp eq <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -27,7 +27,7 @@ define <4 x i16> @cmeq4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmeq v0.4h, v0.4h, v1.4h ; CHECK-NEXT: ret - %tmp3 = icmp eq <4 x i16> %A, %B; + %tmp3 = icmp eq <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -37,7 +37,7 @@ define <8 x i16> @cmeq8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmeq v0.8h, v0.8h, v1.8h ; CHECK-NEXT: ret - %tmp3 = icmp eq <8 x i16> %A, %B; + %tmp3 = icmp eq <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -47,7 +47,7 @@ define <2 x i32> @cmeq2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmeq v0.2s, v0.2s, v1.2s ; CHECK-NEXT: ret - %tmp3 = icmp eq <2 x i32> %A, %B; + %tmp3 = icmp eq <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -57,7 +57,7 @@ define <4 x i32> @cmeq4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmeq v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret - %tmp3 = icmp eq <4 x i32> %A, %B; + %tmp3 = icmp eq <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -67,7 +67,7 @@ define <2 x i64> @cmeq2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmeq v0.2d, v0.2d, v1.2d ; CHECK-NEXT: ret - %tmp3 = icmp eq <2 x i64> %A, %B; + %tmp3 = icmp eq <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -78,7 +78,7 @@ define <8 x i8> @cmne8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK-NEXT: cmeq v0.8b, v0.8b, v1.8b ; CHECK-NEXT: mvn v0.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp ne <8 x i8> %A, %B; + %tmp3 = icmp ne <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -89,7 +89,7 @@ define <16 x i8> @cmne16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK-NEXT: cmeq v0.16b, v0.16b, v1.16b ; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp ne <16 x i8> %A, %B; + %tmp3 = icmp ne <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -100,7 +100,7 @@ define <4 x i16> @cmne4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK-NEXT: cmeq v0.4h, v0.4h, v1.4h ; CHECK-NEXT: mvn v0.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp ne <4 x i16> %A, %B; + %tmp3 = icmp ne <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -111,7 +111,7 @@ define <8 x i16> @cmne8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK-NEXT: cmeq v0.8h, v0.8h, v1.8h ; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp ne <8 x i16> %A, %B; + %tmp3 = icmp ne <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -122,7 +122,7 @@ define <2 x i32> @cmne2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK-NEXT: cmeq v0.2s, v0.2s, v1.2s ; CHECK-NEXT: mvn v0.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp ne <2 x i32> %A, %B; + %tmp3 = icmp ne <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -133,7 +133,7 @@ define <4 x i32> @cmne4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK-NEXT: cmeq v0.4s, v0.4s, v1.4s ; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp ne <4 x i32> %A, %B; + %tmp3 = icmp ne <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -144,7 +144,7 @@ define <2 x i64> @cmne2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK-NEXT: cmeq v0.2d, v0.2d, v1.2d ; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp ne <2 x i64> %A, %B; + %tmp3 = icmp ne <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -154,7 +154,7 @@ define <8 x i8> @cmgt8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.8b, v0.8b, v1.8b ; CHECK-NEXT: ret - %tmp3 = icmp sgt <8 x i8> %A, %B; + %tmp3 = icmp sgt <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -164,7 +164,7 @@ define <16 x i8> @cmgt16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret - %tmp3 = icmp sgt <16 x i8> %A, %B; + %tmp3 = icmp sgt <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -174,7 +174,7 @@ define <4 x i16> @cmgt4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.4h, v0.4h, v1.4h ; CHECK-NEXT: ret - %tmp3 = icmp sgt <4 x i16> %A, %B; + %tmp3 = icmp sgt <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -184,7 +184,7 @@ define <8 x i16> @cmgt8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.8h, v0.8h, v1.8h ; CHECK-NEXT: ret - %tmp3 = icmp sgt <8 x i16> %A, %B; + %tmp3 = icmp sgt <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -194,7 +194,7 @@ define <2 x i32> @cmgt2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.2s, v0.2s, v1.2s ; CHECK-NEXT: ret - %tmp3 = icmp sgt <2 x i32> %A, %B; + %tmp3 = icmp sgt <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -204,7 +204,7 @@ define <4 x i32> @cmgt4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret - %tmp3 = icmp sgt <4 x i32> %A, %B; + %tmp3 = icmp sgt <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -214,7 +214,7 @@ define <2 x i64> @cmgt2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.2d, v0.2d, v1.2d ; CHECK-NEXT: ret - %tmp3 = icmp sgt <2 x i64> %A, %B; + %tmp3 = icmp sgt <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -225,7 +225,7 @@ define <8 x i8> @cmlt8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp slt <8 x i8> %A, %B; + %tmp3 = icmp slt <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -236,7 +236,7 @@ define <16 x i8> @cmlt16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.16b, v1.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp slt <16 x i8> %A, %B; + %tmp3 = icmp slt <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -247,7 +247,7 @@ define <4 x i16> @cmlt4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.4h, v1.4h, v0.4h ; CHECK-NEXT: ret - %tmp3 = icmp slt <4 x i16> %A, %B; + %tmp3 = icmp slt <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -258,7 +258,7 @@ define <8 x i16> @cmlt8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.8h, v1.8h, v0.8h ; CHECK-NEXT: ret - %tmp3 = icmp slt <8 x i16> %A, %B; + %tmp3 = icmp slt <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -269,7 +269,7 @@ define <2 x i32> @cmlt2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.2s, v1.2s, v0.2s ; CHECK-NEXT: ret - %tmp3 = icmp slt <2 x i32> %A, %B; + %tmp3 = icmp slt <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -280,7 +280,7 @@ define <4 x i32> @cmlt4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ret - %tmp3 = icmp slt <4 x i32> %A, %B; + %tmp3 = icmp slt <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -291,7 +291,7 @@ define <2 x i64> @cmlt2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.2d, v1.2d, v0.2d ; CHECK-NEXT: ret - %tmp3 = icmp slt <2 x i64> %A, %B; + %tmp3 = icmp slt <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -301,7 +301,7 @@ define <8 x i8> @cmge8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.8b, v0.8b, v1.8b ; CHECK-NEXT: ret - %tmp3 = icmp sge <8 x i8> %A, %B; + %tmp3 = icmp sge <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -311,7 +311,7 @@ define <16 x i8> @cmge16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret - %tmp3 = icmp sge <16 x i8> %A, %B; + %tmp3 = icmp sge <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -321,7 +321,7 @@ define <4 x i16> @cmge4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.4h, v0.4h, v1.4h ; CHECK-NEXT: ret - %tmp3 = icmp sge <4 x i16> %A, %B; + %tmp3 = icmp sge <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -331,7 +331,7 @@ define <8 x i16> @cmge8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.8h, v0.8h, v1.8h ; CHECK-NEXT: ret - %tmp3 = icmp sge <8 x i16> %A, %B; + %tmp3 = icmp sge <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -341,7 +341,7 @@ define <2 x i32> @cmge2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.2s, v0.2s, v1.2s ; CHECK-NEXT: ret - %tmp3 = icmp sge <2 x i32> %A, %B; + %tmp3 = icmp sge <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -351,7 +351,7 @@ define <4 x i32> @cmge4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret - %tmp3 = icmp sge <4 x i32> %A, %B; + %tmp3 = icmp sge <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -361,7 +361,7 @@ define <2 x i64> @cmge2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.2d, v0.2d, v1.2d ; CHECK-NEXT: ret - %tmp3 = icmp sge <2 x i64> %A, %B; + %tmp3 = icmp sge <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -372,7 +372,7 @@ define <8 x i8> @cmle8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp sle <8 x i8> %A, %B; + %tmp3 = icmp sle <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -383,7 +383,7 @@ define <16 x i8> @cmle16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.16b, v1.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp sle <16 x i8> %A, %B; + %tmp3 = icmp sle <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -394,7 +394,7 @@ define <4 x i16> @cmle4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.4h, v1.4h, v0.4h ; CHECK-NEXT: ret - %tmp3 = icmp sle <4 x i16> %A, %B; + %tmp3 = icmp sle <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -405,7 +405,7 @@ define <8 x i16> @cmle8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.8h, v1.8h, v0.8h ; CHECK-NEXT: ret - %tmp3 = icmp sle <8 x i16> %A, %B; + %tmp3 = icmp sle <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -416,7 +416,7 @@ define <2 x i32> @cmle2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.2s, v1.2s, v0.2s ; CHECK-NEXT: ret - %tmp3 = icmp sle <2 x i32> %A, %B; + %tmp3 = icmp sle <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -427,7 +427,7 @@ define <4 x i32> @cmle4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ret - %tmp3 = icmp sle <4 x i32> %A, %B; + %tmp3 = icmp sle <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -438,7 +438,7 @@ define <2 x i64> @cmle2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.2d, v1.2d, v0.2d ; CHECK-NEXT: ret - %tmp3 = icmp sle <2 x i64> %A, %B; + %tmp3 = icmp sle <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -448,7 +448,7 @@ define <8 x i8> @cmhi8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.8b, v0.8b, v1.8b ; CHECK-NEXT: ret - %tmp3 = icmp ugt <8 x i8> %A, %B; + %tmp3 = icmp ugt <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -458,7 +458,7 @@ define <16 x i8> @cmhi16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret - %tmp3 = icmp ugt <16 x i8> %A, %B; + %tmp3 = icmp ugt <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -468,7 +468,7 @@ define <4 x i16> @cmhi4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.4h, v0.4h, v1.4h ; CHECK-NEXT: ret - %tmp3 = icmp ugt <4 x i16> %A, %B; + %tmp3 = icmp ugt <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -478,7 +478,7 @@ define <8 x i16> @cmhi8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.8h, v0.8h, v1.8h ; CHECK-NEXT: ret - %tmp3 = icmp ugt <8 x i16> %A, %B; + %tmp3 = icmp ugt <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -488,7 +488,7 @@ define <2 x i32> @cmhi2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.2s, v0.2s, v1.2s ; CHECK-NEXT: ret - %tmp3 = icmp ugt <2 x i32> %A, %B; + %tmp3 = icmp ugt <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -498,7 +498,7 @@ define <4 x i32> @cmhi4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret - %tmp3 = icmp ugt <4 x i32> %A, %B; + %tmp3 = icmp ugt <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -508,7 +508,7 @@ define <2 x i64> @cmhi2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.2d, v0.2d, v1.2d ; CHECK-NEXT: ret - %tmp3 = icmp ugt <2 x i64> %A, %B; + %tmp3 = icmp ugt <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -519,7 +519,7 @@ define <8 x i8> @cmlo8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp ult <8 x i8> %A, %B; + %tmp3 = icmp ult <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -530,7 +530,7 @@ define <16 x i8> @cmlo16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.16b, v1.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp ult <16 x i8> %A, %B; + %tmp3 = icmp ult <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -541,7 +541,7 @@ define <4 x i16> @cmlo4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.4h, v1.4h, v0.4h ; CHECK-NEXT: ret - %tmp3 = icmp ult <4 x i16> %A, %B; + %tmp3 = icmp ult <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -552,7 +552,7 @@ define <8 x i16> @cmlo8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.8h, v1.8h, v0.8h ; CHECK-NEXT: ret - %tmp3 = icmp ult <8 x i16> %A, %B; + %tmp3 = icmp ult <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -563,7 +563,7 @@ define <2 x i32> @cmlo2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.2s, v1.2s, v0.2s ; CHECK-NEXT: ret - %tmp3 = icmp ult <2 x i32> %A, %B; + %tmp3 = icmp ult <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -574,7 +574,7 @@ define <4 x i32> @cmlo4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ret - %tmp3 = icmp ult <4 x i32> %A, %B; + %tmp3 = icmp ult <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -585,7 +585,7 @@ define <2 x i64> @cmlo2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.2d, v1.2d, v0.2d ; CHECK-NEXT: ret - %tmp3 = icmp ult <2 x i64> %A, %B; + %tmp3 = icmp ult <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -595,7 +595,7 @@ define <8 x i8> @cmhs8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.8b, v0.8b, v1.8b ; CHECK-NEXT: ret - %tmp3 = icmp uge <8 x i8> %A, %B; + %tmp3 = icmp uge <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -605,7 +605,7 @@ define <16 x i8> @cmhs16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret - %tmp3 = icmp uge <16 x i8> %A, %B; + %tmp3 = icmp uge <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -615,7 +615,7 @@ define <4 x i16> @cmhs4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.4h, v0.4h, v1.4h ; CHECK-NEXT: ret - %tmp3 = icmp uge <4 x i16> %A, %B; + %tmp3 = icmp uge <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -625,7 +625,7 @@ define <8 x i16> @cmhs8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.8h, v0.8h, v1.8h ; CHECK-NEXT: ret - %tmp3 = icmp uge <8 x i16> %A, %B; + %tmp3 = icmp uge <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -635,7 +635,7 @@ define <2 x i32> @cmhs2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.2s, v0.2s, v1.2s ; CHECK-NEXT: ret - %tmp3 = icmp uge <2 x i32> %A, %B; + %tmp3 = icmp uge <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -645,7 +645,7 @@ define <4 x i32> @cmhs4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret - %tmp3 = icmp uge <4 x i32> %A, %B; + %tmp3 = icmp uge <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -655,7 +655,7 @@ define <2 x i64> @cmhs2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.2d, v0.2d, v1.2d ; CHECK-NEXT: ret - %tmp3 = icmp uge <2 x i64> %A, %B; + %tmp3 = icmp uge <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -666,7 +666,7 @@ define <8 x i8> @cmls8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp ule <8 x i8> %A, %B; + %tmp3 = icmp ule <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -677,7 +677,7 @@ define <16 x i8> @cmls16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.16b, v1.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp ule <16 x i8> %A, %B; + %tmp3 = icmp ule <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -688,7 +688,7 @@ define <4 x i16> @cmls4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.4h, v1.4h, v0.4h ; CHECK-NEXT: ret - %tmp3 = icmp ule <4 x i16> %A, %B; + %tmp3 = icmp ule <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -699,7 +699,7 @@ define <8 x i16> @cmls8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.8h, v1.8h, v0.8h ; CHECK-NEXT: ret - %tmp3 = icmp ule <8 x i16> %A, %B; + %tmp3 = icmp ule <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -710,7 +710,7 @@ define <2 x i32> @cmls2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.2s, v1.2s, v0.2s ; CHECK-NEXT: ret - %tmp3 = icmp ule <2 x i32> %A, %B; + %tmp3 = icmp ule <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -721,7 +721,7 @@ define <4 x i32> @cmls4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ret - %tmp3 = icmp ule <4 x i32> %A, %B; + %tmp3 = icmp ule <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -732,7 +732,7 @@ define <2 x i64> @cmls2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.2d, v1.2d, v0.2d ; CHECK-NEXT: ret - %tmp3 = icmp ule <2 x i64> %A, %B; + %tmp3 = icmp ule <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -883,7 +883,7 @@ define <8 x i8> @cmeqz8xi8(<8 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmeq v0.8b, v0.8b, v1.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp eq <8 x i8> %A, zeroinitializer; + %tmp3 = icmp eq <8 x i8> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -899,7 +899,7 @@ define <16 x i8> @cmeqz16xi8(<16 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmeq v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp eq <16 x i8> %A, zeroinitializer; + %tmp3 = icmp eq <16 x i8> %A, zeroinitializer %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -915,7 +915,7 @@ define <4 x i16> @cmeqz4xi16(<4 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmeq v0.4h, v0.4h, v1.4h ; CHECK-GI-NEXT: ret - %tmp3 = icmp eq <4 x i16> %A, zeroinitializer; + %tmp3 = icmp eq <4 x i16> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -931,7 +931,7 @@ define <8 x i16> @cmeqz8xi16(<8 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmeq v0.8h, v0.8h, v1.8h ; CHECK-GI-NEXT: ret - %tmp3 = icmp eq <8 x i16> %A, zeroinitializer; + %tmp3 = icmp eq <8 x i16> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -947,7 +947,7 @@ define <2 x i32> @cmeqz2xi32(<2 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmeq v0.2s, v0.2s, v1.2s ; CHECK-GI-NEXT: ret - %tmp3 = icmp eq <2 x i32> %A, zeroinitializer; + %tmp3 = icmp eq <2 x i32> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -963,7 +963,7 @@ define <4 x i32> @cmeqz4xi32(<4 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmeq v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: ret - %tmp3 = icmp eq <4 x i32> %A, zeroinitializer; + %tmp3 = icmp eq <4 x i32> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -979,7 +979,7 @@ define <2 x i64> @cmeqz2xi64(<2 x i64> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmeq v0.2d, v0.2d, v1.2d ; CHECK-GI-NEXT: ret - %tmp3 = icmp eq <2 x i64> %A, zeroinitializer; + %tmp3 = icmp eq <2 x i64> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -996,7 +996,7 @@ define <8 x i8> @cmgez8xi8(<8 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.8b, v0.8b, v1.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp sge <8 x i8> %A, zeroinitializer; + %tmp3 = icmp sge <8 x i8> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -1012,7 +1012,7 @@ define <16 x i8> @cmgez16xi8(<16 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp sge <16 x i8> %A, zeroinitializer; + %tmp3 = icmp sge <16 x i8> %A, zeroinitializer %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -1028,7 +1028,7 @@ define <4 x i16> @cmgez4xi16(<4 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.4h, v0.4h, v1.4h ; CHECK-GI-NEXT: ret - %tmp3 = icmp sge <4 x i16> %A, zeroinitializer; + %tmp3 = icmp sge <4 x i16> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -1044,7 +1044,7 @@ define <8 x i16> @cmgez8xi16(<8 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.8h, v0.8h, v1.8h ; CHECK-GI-NEXT: ret - %tmp3 = icmp sge <8 x i16> %A, zeroinitializer; + %tmp3 = icmp sge <8 x i16> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -1060,7 +1060,7 @@ define <2 x i32> @cmgez2xi32(<2 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.2s, v0.2s, v1.2s ; CHECK-GI-NEXT: ret - %tmp3 = icmp sge <2 x i32> %A, zeroinitializer; + %tmp3 = icmp sge <2 x i32> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -1076,7 +1076,7 @@ define <4 x i32> @cmgez4xi32(<4 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: ret - %tmp3 = icmp sge <4 x i32> %A, zeroinitializer; + %tmp3 = icmp sge <4 x i32> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -1092,7 +1092,7 @@ define <2 x i64> @cmgez2xi64(<2 x i64> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.2d, v0.2d, v1.2d ; CHECK-GI-NEXT: ret - %tmp3 = icmp sge <2 x i64> %A, zeroinitializer; + %tmp3 = icmp sge <2 x i64> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -1210,6 +1210,118 @@ define <2 x i64> @cmgez2xi64_alt(<2 x i64> %A) { ret <2 x i64> %not } +define <8 x i8> @cmgez8xi8_alt2(<8 x i8> %A) { +; CHECK-SD-LABEL: cmgez8xi8_alt2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmge v0.8b, v0.8b, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmgez8xi8_alt2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi d1, #0xffffffffffffffff +; CHECK-GI-NEXT: cmgt v0.8b, v0.8b, v1.8b +; CHECK-GI-NEXT: ret + %tmp3 = icmp sgt <8 x i8> %A, + %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> + ret <8 x i8> %tmp4 +} + +define <16 x i8> @cmgez16xi8_alt2(<16 x i8> %A) { +; CHECK-SD-LABEL: cmgez16xi8_alt2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmge v0.16b, v0.16b, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmgez16xi8_alt2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.2d, #0xffffffffffffffff +; CHECK-GI-NEXT: cmgt v0.16b, v0.16b, v1.16b +; CHECK-GI-NEXT: ret + %tmp3 = icmp sgt <16 x i8> %A, + %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> + ret <16 x i8> %tmp4 +} + +define <4 x i16> @cmgez4xi16_alt2(<4 x i16> %A) { +; CHECK-SD-LABEL: cmgez4xi16_alt2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmge v0.4h, v0.4h, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmgez4xi16_alt2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi d1, #0xffffffffffffffff +; CHECK-GI-NEXT: cmgt v0.4h, v0.4h, v1.4h +; CHECK-GI-NEXT: ret + %tmp3 = icmp sgt <4 x i16> %A, + %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> + ret <4 x i16> %tmp4 +} + +define <8 x i16> @cmgez8xi16_alt2(<8 x i16> %A) { +; CHECK-SD-LABEL: cmgez8xi16_alt2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmge v0.8h, v0.8h, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmgez8xi16_alt2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.2d, #0xffffffffffffffff +; CHECK-GI-NEXT: cmgt v0.8h, v0.8h, v1.8h +; CHECK-GI-NEXT: ret + %tmp3 = icmp sgt <8 x i16> %A, + %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> + ret <8 x i16> %tmp4 +} + +define <2 x i32> @cmgez2xi32_alt2(<2 x i32> %A) { +; CHECK-SD-LABEL: cmgez2xi32_alt2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmge v0.2s, v0.2s, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmgez2xi32_alt2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi d1, #0xffffffffffffffff +; CHECK-GI-NEXT: cmgt v0.2s, v0.2s, v1.2s +; CHECK-GI-NEXT: ret + %tmp3 = icmp sgt <2 x i32> %A, + %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> + ret <2 x i32> %tmp4 +} + +define <4 x i32> @cmgez4xi32_alt2(<4 x i32> %A) { +; CHECK-SD-LABEL: cmgez4xi32_alt2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmge v0.4s, v0.4s, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmgez4xi32_alt2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.2d, #0xffffffffffffffff +; CHECK-GI-NEXT: cmgt v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: ret + %tmp3 = icmp sgt <4 x i32> %A, + %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> + ret <4 x i32> %tmp4 +} + +define <2 x i64> @cmgez2xi64_alt2(<2 x i64> %A) { +; CHECK-SD-LABEL: cmgez2xi64_alt2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmge v0.2d, v0.2d, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmgez2xi64_alt2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.2d, #0xffffffffffffffff +; CHECK-GI-NEXT: cmgt v0.2d, v0.2d, v1.2d +; CHECK-GI-NEXT: ret + %tmp3 = icmp sgt <2 x i64> %A, + %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> + ret <2 x i64> %tmp4 +} + define <8 x i8> @cmgtz8xi8(<8 x i8> %A) { ; CHECK-SD-LABEL: cmgtz8xi8: @@ -1222,7 +1334,7 @@ define <8 x i8> @cmgtz8xi8(<8 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.8b, v0.8b, v1.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp sgt <8 x i8> %A, zeroinitializer; + %tmp3 = icmp sgt <8 x i8> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -1238,7 +1350,7 @@ define <16 x i8> @cmgtz16xi8(<16 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp sgt <16 x i8> %A, zeroinitializer; + %tmp3 = icmp sgt <16 x i8> %A, zeroinitializer %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -1254,7 +1366,7 @@ define <4 x i16> @cmgtz4xi16(<4 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.4h, v0.4h, v1.4h ; CHECK-GI-NEXT: ret - %tmp3 = icmp sgt <4 x i16> %A, zeroinitializer; + %tmp3 = icmp sgt <4 x i16> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -1270,7 +1382,7 @@ define <8 x i16> @cmgtz8xi16(<8 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.8h, v0.8h, v1.8h ; CHECK-GI-NEXT: ret - %tmp3 = icmp sgt <8 x i16> %A, zeroinitializer; + %tmp3 = icmp sgt <8 x i16> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -1286,7 +1398,7 @@ define <2 x i32> @cmgtz2xi32(<2 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.2s, v0.2s, v1.2s ; CHECK-GI-NEXT: ret - %tmp3 = icmp sgt <2 x i32> %A, zeroinitializer; + %tmp3 = icmp sgt <2 x i32> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -1302,7 +1414,7 @@ define <4 x i32> @cmgtz4xi32(<4 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: ret - %tmp3 = icmp sgt <4 x i32> %A, zeroinitializer; + %tmp3 = icmp sgt <4 x i32> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -1318,7 +1430,7 @@ define <2 x i64> @cmgtz2xi64(<2 x i64> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.2d, v0.2d, v1.2d ; CHECK-GI-NEXT: ret - %tmp3 = icmp sgt <2 x i64> %A, zeroinitializer; + %tmp3 = icmp sgt <2 x i64> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -1334,7 +1446,7 @@ define <8 x i8> @cmlez8xi8(<8 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.8b, v1.8b, v0.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp sle <8 x i8> %A, zeroinitializer; + %tmp3 = icmp sle <8 x i8> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -1350,7 +1462,7 @@ define <16 x i8> @cmlez16xi8(<16 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.16b, v1.16b, v0.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp sle <16 x i8> %A, zeroinitializer; + %tmp3 = icmp sle <16 x i8> %A, zeroinitializer %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -1366,7 +1478,7 @@ define <4 x i16> @cmlez4xi16(<4 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.4h, v1.4h, v0.4h ; CHECK-GI-NEXT: ret - %tmp3 = icmp sle <4 x i16> %A, zeroinitializer; + %tmp3 = icmp sle <4 x i16> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -1382,7 +1494,7 @@ define <8 x i16> @cmlez8xi16(<8 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.8h, v1.8h, v0.8h ; CHECK-GI-NEXT: ret - %tmp3 = icmp sle <8 x i16> %A, zeroinitializer; + %tmp3 = icmp sle <8 x i16> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -1398,7 +1510,7 @@ define <2 x i32> @cmlez2xi32(<2 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.2s, v1.2s, v0.2s ; CHECK-GI-NEXT: ret - %tmp3 = icmp sle <2 x i32> %A, zeroinitializer; + %tmp3 = icmp sle <2 x i32> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -1414,7 +1526,7 @@ define <4 x i32> @cmlez4xi32(<4 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.4s, v1.4s, v0.4s ; CHECK-GI-NEXT: ret - %tmp3 = icmp sle <4 x i32> %A, zeroinitializer; + %tmp3 = icmp sle <4 x i32> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -1430,7 +1542,120 @@ define <2 x i64> @cmlez2xi64(<2 x i64> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.2d, v1.2d, v0.2d ; CHECK-GI-NEXT: ret - %tmp3 = icmp sle <2 x i64> %A, zeroinitializer; + %tmp3 = icmp sle <2 x i64> %A, zeroinitializer + %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> + ret <2 x i64> %tmp4 +} + +define <8 x i8> @cmlez8xi8_alt(<8 x i8> %A) { +; CHECK-SD-LABEL: cmlez8xi8_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmle v0.8b, v0.8b, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmlez8xi8_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.8b, #1 +; CHECK-GI-NEXT: cmgt v0.8b, v1.8b, v0.8b +; CHECK-GI-NEXT: ret + %tmp3 = icmp slt <8 x i8> %A, + %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> + ret <8 x i8> %tmp4 +} + +define <16 x i8> @cmlez16xi8_alt(<16 x i8> %A) { +; CHECK-SD-LABEL: cmlez16xi8_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmle v0.16b, v0.16b, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmlez16xi8_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.16b, #1 +; CHECK-GI-NEXT: cmgt v0.16b, v1.16b, v0.16b +; CHECK-GI-NEXT: ret + %tmp3 = icmp slt <16 x i8> %A, + %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> + ret <16 x i8> %tmp4 +} + +define <4 x i16> @cmlez4xi16_alt(<4 x i16> %A) { +; CHECK-SD-LABEL: cmlez4xi16_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmle v0.4h, v0.4h, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmlez4xi16_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.4h, #1 +; CHECK-GI-NEXT: cmgt v0.4h, v1.4h, v0.4h +; CHECK-GI-NEXT: ret + %tmp3 = icmp slt <4 x i16> %A, + %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> + ret <4 x i16> %tmp4 +} + +define <8 x i16> @cmlez8xi16_alt(<8 x i16> %A) { +; CHECK-SD-LABEL: cmlez8xi16_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmle v0.8h, v0.8h, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmlez8xi16_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.8h, #1 +; CHECK-GI-NEXT: cmgt v0.8h, v1.8h, v0.8h +; CHECK-GI-NEXT: ret + %tmp3 = icmp slt <8 x i16> %A, + %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> + ret <8 x i16> %tmp4 +} + +define <2 x i32> @cmlez2xi32_alt(<2 x i32> %A) { +; CHECK-SD-LABEL: cmlez2xi32_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmle v0.2s, v0.2s, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmlez2xi32_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.2s, #1 +; CHECK-GI-NEXT: cmgt v0.2s, v1.2s, v0.2s +; CHECK-GI-NEXT: ret + %tmp3 = icmp slt <2 x i32> %A, + %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> + ret <2 x i32> %tmp4 +} + +define <4 x i32> @cmlez4xi32_alt(<4 x i32> %A) { +; CHECK-SD-LABEL: cmlez4xi32_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmle v0.4s, v0.4s, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmlez4xi32_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.4s, #1 +; CHECK-GI-NEXT: cmgt v0.4s, v1.4s, v0.4s +; CHECK-GI-NEXT: ret + %tmp3 = icmp slt <4 x i32> %A, + %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> + ret <4 x i32> %tmp4 +} + +define <2 x i64> @cmlez2xi64_alt(<2 x i64> %A) { +; CHECK-SD-LABEL: cmlez2xi64_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmle v0.2d, v0.2d, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmlez2xi64_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: adrp x8, .LCPI125_0 +; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI125_0] +; CHECK-GI-NEXT: cmgt v0.2d, v1.2d, v0.2d +; CHECK-GI-NEXT: ret + %tmp3 = icmp slt <2 x i64> %A, %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -1446,7 +1671,7 @@ define <8 x i8> @cmltz8xi8(<8 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.8b, v1.8b, v0.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp slt <8 x i8> %A, zeroinitializer; + %tmp3 = icmp slt <8 x i8> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -1462,7 +1687,7 @@ define <16 x i8> @cmltz16xi8(<16 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.16b, v1.16b, v0.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp slt <16 x i8> %A, zeroinitializer; + %tmp3 = icmp slt <16 x i8> %A, zeroinitializer %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -1478,7 +1703,7 @@ define <4 x i16> @cmltz4xi16(<4 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.4h, v1.4h, v0.4h ; CHECK-GI-NEXT: ret - %tmp3 = icmp slt <4 x i16> %A, zeroinitializer; + %tmp3 = icmp slt <4 x i16> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -1494,7 +1719,7 @@ define <8 x i16> @cmltz8xi16(<8 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.8h, v1.8h, v0.8h ; CHECK-GI-NEXT: ret - %tmp3 = icmp slt <8 x i16> %A, zeroinitializer; + %tmp3 = icmp slt <8 x i16> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -1510,7 +1735,7 @@ define <2 x i32> @cmltz2xi32(<2 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.2s, v1.2s, v0.2s ; CHECK-GI-NEXT: ret - %tmp3 = icmp slt <2 x i32> %A, zeroinitializer; + %tmp3 = icmp slt <2 x i32> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -1526,7 +1751,7 @@ define <4 x i32> @cmltz4xi32(<4 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.4s, v1.4s, v0.4s ; CHECK-GI-NEXT: ret - %tmp3 = icmp slt <4 x i32> %A, zeroinitializer; + %tmp3 = icmp slt <4 x i32> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -1542,11 +1767,109 @@ define <2 x i64> @cmltz2xi64(<2 x i64> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.2d, v1.2d, v0.2d ; CHECK-GI-NEXT: ret - %tmp3 = icmp slt <2 x i64> %A, zeroinitializer; + %tmp3 = icmp slt <2 x i64> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } +define <8 x i8> @cmltz8xi8_alt(<8 x i8> %A) { +; CHECK-SD-LABEL: cmltz8xi8_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmlt v0.8b, v0.8b, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmltz8xi8_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: sshr v0.8b, v0.8b, #7 +; CHECK-GI-NEXT: ret + %A.lobit = ashr <8 x i8> %A, + ret <8 x i8> %A.lobit +} + +define <16 x i8> @cmltz16xi8_alt(<16 x i8> %A) { +; CHECK-SD-LABEL: cmltz16xi8_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmlt v0.16b, v0.16b, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmltz16xi8_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: sshr v0.16b, v0.16b, #7 +; CHECK-GI-NEXT: ret + %A.lobit = ashr <16 x i8> %A, + ret <16 x i8> %A.lobit +} + +define <4 x i16> @cmltz4xi16_alt(<4 x i16> %A) { +; CHECK-SD-LABEL: cmltz4xi16_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmlt v0.4h, v0.4h, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmltz4xi16_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: sshr v0.4h, v0.4h, #15 +; CHECK-GI-NEXT: ret + %A.lobit = ashr <4 x i16> %A, + ret <4 x i16> %A.lobit +} + +define <8 x i16> @cmltz8xi16_alt(<8 x i16> %A) { +; CHECK-SD-LABEL: cmltz8xi16_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmlt v0.8h, v0.8h, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmltz8xi16_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: sshr v0.8h, v0.8h, #15 +; CHECK-GI-NEXT: ret + %A.lobit = ashr <8 x i16> %A, + ret <8 x i16> %A.lobit +} + +define <2 x i32> @cmltz2xi32_alt(<2 x i32> %A) { +; CHECK-SD-LABEL: cmltz2xi32_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmlt v0.2s, v0.2s, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmltz2xi32_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: sshr v0.2s, v0.2s, #31 +; CHECK-GI-NEXT: ret + %A.lobit = ashr <2 x i32> %A, + ret <2 x i32> %A.lobit +} + +define <4 x i32> @cmltz4xi32_alt(<4 x i32> %A) { +; CHECK-SD-LABEL: cmltz4xi32_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmlt v0.4s, v0.4s, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmltz4xi32_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: sshr v0.4s, v0.4s, #31 +; CHECK-GI-NEXT: ret + %A.lobit = ashr <4 x i32> %A, + ret <4 x i32> %A.lobit +} + +define <2 x i64> @cmltz2xi64_alt(<2 x i64> %A) { +; CHECK-SD-LABEL: cmltz2xi64_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmlt v0.2d, v0.2d, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmltz2xi64_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: sshr v0.2d, v0.2d, #63 +; CHECK-GI-NEXT: ret + %A.lobit = ashr <2 x i64> %A, + ret <2 x i64> %A.lobit +} + define <8 x i8> @cmneqz8xi8(<8 x i8> %A) { ; CHECK-SD-LABEL: cmneqz8xi8: ; CHECK-SD: // %bb.0: @@ -1559,7 +1882,7 @@ define <8 x i8> @cmneqz8xi8(<8 x i8> %A) { ; CHECK-GI-NEXT: cmeq v0.8b, v0.8b, v1.8b ; CHECK-GI-NEXT: mvn v0.8b, v0.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp ne <8 x i8> %A, zeroinitializer; + %tmp3 = icmp ne <8 x i8> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -1576,7 +1899,7 @@ define <16 x i8> @cmneqz16xi8(<16 x i8> %A) { ; CHECK-GI-NEXT: cmeq v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: mvn v0.16b, v0.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp ne <16 x i8> %A, zeroinitializer; + %tmp3 = icmp ne <16 x i8> %A, zeroinitializer %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -1593,7 +1916,7 @@ define <4 x i16> @cmneqz4xi16(<4 x i16> %A) { ; CHECK-GI-NEXT: cmeq v0.4h, v0.4h, v1.4h ; CHECK-GI-NEXT: mvn v0.8b, v0.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp ne <4 x i16> %A, zeroinitializer; + %tmp3 = icmp ne <4 x i16> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -1610,7 +1933,7 @@ define <8 x i16> @cmneqz8xi16(<8 x i16> %A) { ; CHECK-GI-NEXT: cmeq v0.8h, v0.8h, v1.8h ; CHECK-GI-NEXT: mvn v0.16b, v0.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp ne <8 x i16> %A, zeroinitializer; + %tmp3 = icmp ne <8 x i16> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -1627,7 +1950,7 @@ define <2 x i32> @cmneqz2xi32(<2 x i32> %A) { ; CHECK-GI-NEXT: cmeq v0.2s, v0.2s, v1.2s ; CHECK-GI-NEXT: mvn v0.8b, v0.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp ne <2 x i32> %A, zeroinitializer; + %tmp3 = icmp ne <2 x i32> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -1644,7 +1967,7 @@ define <4 x i32> @cmneqz4xi32(<4 x i32> %A) { ; CHECK-GI-NEXT: cmeq v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: mvn v0.16b, v0.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp ne <4 x i32> %A, zeroinitializer; + %tmp3 = icmp ne <4 x i32> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -1661,7 +1984,7 @@ define <2 x i64> @cmneqz2xi64(<2 x i64> %A) { ; CHECK-GI-NEXT: cmeq v0.2d, v0.2d, v1.2d ; CHECK-GI-NEXT: mvn v0.16b, v0.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp ne <2 x i64> %A, zeroinitializer; + %tmp3 = icmp ne <2 x i64> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -1742,16 +2065,10 @@ define <2 x i64> @cmhsz2xi64(<2 x i64> %A) { ; ; CHECK-GI-LABEL: cmhsz2xi64: ; CHECK-GI: // %bb.0: -; CHECK-GI-NEXT: adrp x8, .LCPI132_0 -; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI132_0] +; CHECK-GI-NEXT: adrp x8, .LCPI153_0 +; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI153_0] ; CHECK-GI-NEXT: cmhs v0.2d, v0.2d, v1.2d ; CHECK-GI-NEXT: ret -; GISEL-LABEL: cmhsz2xi64: -; GISEL: // %bb.0: -; GISEL-NEXT: adrp x8, .LCPI132_0 -; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI132_0] -; GISEL-NEXT: cmhs v0.2d, v0.2d, v1.2d -; GISEL-NEXT: ret %tmp3 = icmp uge <2 x i64> %A, %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 @@ -1834,16 +2151,10 @@ define <2 x i64> @cmhiz2xi64(<2 x i64> %A) { ; ; CHECK-GI-LABEL: cmhiz2xi64: ; CHECK-GI: // %bb.0: -; CHECK-GI-NEXT: adrp x8, .LCPI139_0 -; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI139_0] +; CHECK-GI-NEXT: adrp x8, .LCPI160_0 +; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI160_0] ; CHECK-GI-NEXT: cmhi v0.2d, v0.2d, v1.2d ; CHECK-GI-NEXT: ret -; GISEL-LABEL: cmhiz2xi64: -; GISEL: // %bb.0: -; GISEL-NEXT: adrp x8, .LCPI139_0 -; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI139_0] -; GISEL-NEXT: cmhi v0.2d, v0.2d, v1.2d -; GISEL-NEXT: ret %tmp3 = icmp ugt <2 x i64> %A, %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 @@ -1856,7 +2167,7 @@ define <8 x i8> @cmlsz8xi8(<8 x i8> %A) { ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cmhs v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp ule <8 x i8> %A, zeroinitializer; + %tmp3 = icmp ule <8 x i8> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -1868,7 +2179,7 @@ define <16 x i8> @cmlsz16xi8(<16 x i8> %A) { ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cmhs v0.16b, v1.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp ule <16 x i8> %A, zeroinitializer; + %tmp3 = icmp ule <16 x i8> %A, zeroinitializer %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -1880,7 +2191,7 @@ define <4 x i16> @cmlsz4xi16(<4 x i16> %A) { ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cmhs v0.4h, v1.4h, v0.4h ; CHECK-NEXT: ret - %tmp3 = icmp ule <4 x i16> %A, zeroinitializer; + %tmp3 = icmp ule <4 x i16> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -1892,7 +2203,7 @@ define <8 x i16> @cmlsz8xi16(<8 x i16> %A) { ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cmhs v0.8h, v1.8h, v0.8h ; CHECK-NEXT: ret - %tmp3 = icmp ule <8 x i16> %A, zeroinitializer; + %tmp3 = icmp ule <8 x i16> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -1904,7 +2215,7 @@ define <2 x i32> @cmlsz2xi32(<2 x i32> %A) { ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cmhs v0.2s, v1.2s, v0.2s ; CHECK-NEXT: ret - %tmp3 = icmp ule <2 x i32> %A, zeroinitializer; + %tmp3 = icmp ule <2 x i32> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -1916,7 +2227,7 @@ define <4 x i32> @cmlsz4xi32(<4 x i32> %A) { ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cmhs v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ret - %tmp3 = icmp ule <4 x i32> %A, zeroinitializer; + %tmp3 = icmp ule <4 x i32> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -1928,7 +2239,7 @@ define <2 x i64> @cmlsz2xi64(<2 x i64> %A) { ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cmhs v0.2d, v1.2d, v0.2d ; CHECK-NEXT: ret - %tmp3 = icmp ule <2 x i64> %A, zeroinitializer; + %tmp3 = icmp ule <2 x i64> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -2016,16 +2327,10 @@ define <2 x i64> @cmloz2xi64(<2 x i64> %A) { ; ; CHECK-GI-LABEL: cmloz2xi64: ; CHECK-GI: // %bb.0: -; CHECK-GI-NEXT: adrp x8, .LCPI153_0 -; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI153_0] +; CHECK-GI-NEXT: adrp x8, .LCPI174_0 +; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI174_0] ; CHECK-GI-NEXT: cmhi v0.2d, v1.2d, v0.2d ; CHECK-GI-NEXT: ret -; GISEL-LABEL: cmloz2xi64: -; GISEL: // %bb.0: -; GISEL-NEXT: adrp x8, .LCPI153_0 -; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI153_0] -; GISEL-NEXT: cmhi v0.2d, v1.2d, v0.2d -; GISEL-NEXT: ret %tmp3 = icmp ult <2 x i64> %A, %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 @@ -4175,13 +4480,6 @@ define <4 x i32> @fcmule4xfloat_fast_zext(<4 x float> %A, <4 x float> %B) { ; CHECK-GI-NEXT: fcmgt v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: bic v0.16b, v2.16b, v0.16b ; CHECK-GI-NEXT: ret -; GISEL-LABEL: fcmule4xfloat_fast_zext: -; GISEL: // %bb.0: -; GISEL-NEXT: fcmgt v0.4s, v0.4s, v1.4s -; GISEL-NEXT: adrp x8, .LCPI322_0 -; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI322_0] -; GISEL-NEXT: bic v0.16b, v1.16b, v0.16b -; GISEL-NEXT: ret %tmp3 = fcmp fast ule <4 x float> %A, %B %tmp4 = zext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 diff --git a/llvm/test/CodeGen/AArch64/signbit-shift.ll b/llvm/test/CodeGen/AArch64/signbit-shift.ll index cb758f8a6202b7b25901b114fbacebb891fed745..253ea1cab91fb64007cde1334c2194d3b643b79b 100644 --- a/llvm/test/CodeGen/AArch64/signbit-shift.ll +++ b/llvm/test/CodeGen/AArch64/signbit-shift.ll @@ -29,10 +29,9 @@ define i32 @add_zext_ifpos(i32 %x) { define <4 x i32> @add_zext_ifpos_vec_splat(<4 x i32> %x) { ; CHECK-LABEL: add_zext_ifpos_vec_splat: ; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0xffffffffffffffff -; CHECK-NEXT: movi v2.4s, #41 -; CHECK-NEXT: cmgt v0.4s, v0.4s, v1.4s -; CHECK-NEXT: sub v0.4s, v2.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #41 +; CHECK-NEXT: cmge v0.4s, v0.4s, #0 +; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ret %c = icmp sgt <4 x i32> %x, %e = zext <4 x i1> %c to <4 x i32> @@ -43,7 +42,7 @@ define <4 x i32> @add_zext_ifpos_vec_splat(<4 x i32> %x) { define i32 @sel_ifpos_tval_bigger(i32 %x) { ; CHECK-LABEL: sel_ifpos_tval_bigger: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #41 +; CHECK-NEXT: mov w8, #41 // =0x29 ; CHECK-NEXT: cmp w0, #0 ; CHECK-NEXT: cinc w0, w8, ge ; CHECK-NEXT: ret @@ -78,10 +77,9 @@ define i32 @add_sext_ifpos(i32 %x) { define <4 x i32> @add_sext_ifpos_vec_splat(<4 x i32> %x) { ; CHECK-LABEL: add_sext_ifpos_vec_splat: ; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0xffffffffffffffff -; CHECK-NEXT: movi v2.4s, #42 -; CHECK-NEXT: cmgt v0.4s, v0.4s, v1.4s -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: movi v1.4s, #42 +; CHECK-NEXT: cmge v0.4s, v0.4s, #0 +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret %c = icmp sgt <4 x i32> %x, %e = sext <4 x i1> %c to <4 x i32> @@ -92,7 +90,7 @@ define <4 x i32> @add_sext_ifpos_vec_splat(<4 x i32> %x) { define i32 @sel_ifpos_fval_bigger(i32 %x) { ; CHECK-LABEL: sel_ifpos_fval_bigger: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #41 +; CHECK-NEXT: mov w8, #41 // =0x29 ; CHECK-NEXT: cmp w0, #0 ; CHECK-NEXT: cinc w0, w8, lt ; CHECK-NEXT: ret @@ -128,7 +126,7 @@ define i32 @add_zext_ifneg(i32 %x) { define i32 @sel_ifneg_tval_bigger(i32 %x) { ; CHECK-LABEL: sel_ifneg_tval_bigger: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #41 +; CHECK-NEXT: mov w8, #41 // =0x29 ; CHECK-NEXT: cmp w0, #0 ; CHECK-NEXT: cinc w0, w8, lt ; CHECK-NEXT: ret @@ -162,7 +160,7 @@ define i32 @add_sext_ifneg(i32 %x) { define i32 @sel_ifneg_fval_bigger(i32 %x) { ; CHECK-LABEL: sel_ifneg_fval_bigger: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #41 +; CHECK-NEXT: mov w8, #41 // =0x29 ; CHECK-NEXT: cmp w0, #0 ; CHECK-NEXT: cinc w0, w8, ge ; CHECK-NEXT: ret @@ -199,7 +197,7 @@ define <4 x i32> @add_lshr_not_vec_splat(<4 x i32> %x) { define i32 @sub_lshr_not(i32 %x) { ; CHECK-LABEL: sub_lshr_not: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #42 +; CHECK-NEXT: mov w8, #42 // =0x2a ; CHECK-NEXT: bfxil w8, w0, #31, #1 ; CHECK-NEXT: mov w0, w8 ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/stack-probing-64k.ll b/llvm/test/CodeGen/AArch64/stack-probing-64k.ll index 945c271d375001b4353b7ac2ef9102edc3819657..2f15e317a7f58b224a081e130b0e4c63a6911e7b 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing-64k.ll +++ b/llvm/test/CodeGen/AArch64/stack-probing-64k.ll @@ -313,7 +313,7 @@ define void @static_16_align_131072(ptr %out) #0 { ; CHECK-NEXT: b .LBB9_1 ; CHECK-NEXT: .LBB9_3: // %entry ; CHECK-NEXT: mov sp, x9 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: mov x8, sp ; CHECK-NEXT: str x8, [x0] ; CHECK-NEXT: mov sp, x29 diff --git a/llvm/test/CodeGen/AArch64/stack-probing-dynamic.ll b/llvm/test/CodeGen/AArch64/stack-probing-dynamic.ll index d247ed1b5997750b14e40c6d877891d7bff2a557..a3b8df487ed4888979aa5c6dabcb64160a063dd3 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing-dynamic.ll +++ b/llvm/test/CodeGen/AArch64/stack-probing-dynamic.ll @@ -28,7 +28,7 @@ define void @dynamic(i64 %size, ptr %out) #0 { ; CHECK-NEXT: b .LBB0_1 ; CHECK-NEXT: .LBB0_3: ; CHECK-NEXT: mov sp, x8 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: mov sp, x29 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 @@ -72,7 +72,7 @@ define void @dynamic_fixed(i64 %size, ptr %out1, ptr %out2) #0 { ; CHECK-NEXT: b .LBB1_1 ; CHECK-NEXT: .LBB1_3: ; CHECK-NEXT: mov sp, x8 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: str x8, [x2] ; CHECK-NEXT: mov sp, x29 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 @@ -122,7 +122,7 @@ define void @dynamic_align_64(i64 %size, ptr %out) #0 { ; CHECK-NEXT: b .LBB2_1 ; CHECK-NEXT: .LBB2_3: ; CHECK-NEXT: mov sp, x8 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: mov sp, x29 ; CHECK-NEXT: .cfi_def_cfa wsp, 32 @@ -167,7 +167,7 @@ define void @dynamic_align_8192(i64 %size, ptr %out) #0 { ; CHECK-NEXT: mov sp, x9 ; CHECK-NEXT: add x9, x0, #15 ; CHECK-NEXT: mov x8, sp -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: and x9, x9, #0xfffffffffffffff0 ; CHECK-NEXT: mov x19, sp ; CHECK-NEXT: sub x8, x8, x9 @@ -181,7 +181,7 @@ define void @dynamic_align_8192(i64 %size, ptr %out) #0 { ; CHECK-NEXT: b .LBB3_4 ; CHECK-NEXT: .LBB3_6: ; CHECK-NEXT: mov sp, x8 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: mov sp, x29 ; CHECK-NEXT: .cfi_def_cfa wsp, 32 @@ -221,7 +221,7 @@ define void @dynamic_64k_guard(i64 %size, ptr %out) #0 "stack-probe-size"="65536 ; CHECK-NEXT: b .LBB4_1 ; CHECK-NEXT: .LBB4_3: ; CHECK-NEXT: mov sp, x8 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: mov sp, x29 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 @@ -265,7 +265,7 @@ define void @no_reserved_call_frame(i64 %n) #0 { ; CHECK-NEXT: b .LBB5_1 ; CHECK-NEXT: .LBB5_3: // %entry ; CHECK-NEXT: mov sp, x0 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: sub sp, sp, #1104 ; CHECK-NEXT: str xzr, [sp] ; CHECK-NEXT: bl callee_stack_args @@ -344,7 +344,7 @@ define void @dynamic_sve(i64 %size, ptr %out) #0 "target-features"="+sve" { ; CHECK-NEXT: b .LBB7_1 ; CHECK-NEXT: .LBB7_3: ; CHECK-NEXT: mov sp, x8 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: mov sp, x29 ; CHECK-NEXT: .cfi_def_cfa wsp, 32 diff --git a/llvm/test/CodeGen/AArch64/stack-probing-sve.ll b/llvm/test/CodeGen/AArch64/stack-probing-sve.ll index 4dad104e66f20d560efc77539d08ff6029ae668e..03a9220ebfddc6421466d24e3b5d8ee961c63954 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing-sve.ll +++ b/llvm/test/CodeGen/AArch64/stack-probing-sve.ll @@ -115,7 +115,7 @@ define void @sve_17_vector(ptr %out) #0 { ; CHECK-NEXT: b .LBB3_1 ; CHECK-NEXT: .LBB3_3: // %entry ; CHECK-NEXT: mov sp, x9 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: .cfi_def_cfa_register wsp ; CHECK-NEXT: addvl sp, sp, #17 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 @@ -351,7 +351,7 @@ define void @sve_16v_1p_csr( %a) #0 { ; CHECK-NEXT: b .LBB9_1 ; CHECK-NEXT: .LBB9_3: // %entry ; CHECK-NEXT: mov sp, x9 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: .cfi_def_cfa_register wsp ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill @@ -467,7 +467,7 @@ define void @sve_1_vector_4096_arr(ptr %out) #0 { ; CHECK-NEXT: b .LBB11_1 ; CHECK-NEXT: .LBB11_3: // %entry ; CHECK-NEXT: mov sp, x9 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: .cfi_def_cfa_register wsp ; CHECK-NEXT: addvl sp, sp, #31 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0f, 0x8f, 0x00, 0x11, 0x90, 0xe0, 0x00, 0x22, 0x11, 0x88, 0x02, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 12304 + 264 * VG @@ -516,7 +516,7 @@ define void @sve_1_vector_16_arr_align_8192(ptr %out) #0 { ; CHECK-NEXT: b .LBB12_1 ; CHECK-NEXT: .LBB12_3: // %entry ; CHECK-NEXT: mov sp, x9 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: mov sp, x29 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload @@ -616,7 +616,7 @@ define void @sve_1028_64k_guard(ptr %out) #0 "stack-probe-size"="65536" { ; CHECK-NEXT: b .LBB14_1 ; CHECK-NEXT: .LBB14_3: // %entry ; CHECK-NEXT: mov sp, x9 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: .cfi_def_cfa_register wsp ; CHECK-NEXT: addvl sp, sp, #31 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x0e, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1808 * VG diff --git a/llvm/test/CodeGen/AArch64/stack-probing.ll b/llvm/test/CodeGen/AArch64/stack-probing.ll index 5c5d9321a56e581e38fa1633881b76da19fb2e4e..df5408de5bab0a77bce18d151957782e014329d4 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing.ll +++ b/llvm/test/CodeGen/AArch64/stack-probing.ll @@ -400,7 +400,7 @@ define void @static_16_align_8192(ptr %out) #0 { ; CHECK-NEXT: b .LBB13_1 ; CHECK-NEXT: .LBB13_3: // %entry ; CHECK-NEXT: mov sp, x9 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: mov x8, sp ; CHECK-NEXT: str x8, [x0] ; CHECK-NEXT: mov sp, x29 diff --git a/llvm/test/CodeGen/AArch64/vselect-ext.ll b/llvm/test/CodeGen/AArch64/vselect-ext.ll index 0ae09ebe916302162a0ba0e7c83a3aa9ccf36569..b80955665c74f1edc9d57a3be64749083c437417 100644 --- a/llvm/test/CodeGen/AArch64/vselect-ext.ll +++ b/llvm/test/CodeGen/AArch64/vselect-ext.ll @@ -543,15 +543,14 @@ entry: define <16 x i32> @same_zext_used_in_cmp_signed_pred_and_select_can_convert_to_unsigned_pred(<16 x i8> %a) { ; CHECK-LABEL: same_zext_used_in_cmp_signed_pred_and_select_can_convert_to_unsigned_pred: ; CHECK: ; %bb.0: ; %entry -; CHECK-NEXT: movi.2d v1, #0xffffffffffffffff +; CHECK-NEXT: cmge.16b v1, v0, #0 ; CHECK-NEXT: ushll.8h v2, v0, #0 -; CHECK-NEXT: ushll.4s v4, v2, #0 -; CHECK-NEXT: ushll2.4s v2, v2, #0 -; CHECK-NEXT: cmgt.16b v1, v0, v1 ; CHECK-NEXT: ushll2.8h v0, v0, #0 ; CHECK-NEXT: sshll.8h v3, v1, #0 ; CHECK-NEXT: sshll2.8h v1, v1, #0 +; CHECK-NEXT: ushll.4s v4, v2, #0 ; CHECK-NEXT: ushll.4s v5, v0, #0 +; CHECK-NEXT: ushll2.4s v2, v2, #0 ; CHECK-NEXT: ushll2.4s v6, v0, #0 ; CHECK-NEXT: sshll.4s v0, v3, #0 ; CHECK-NEXT: sshll.4s v7, v1, #0 @@ -574,52 +573,51 @@ define void @extension_in_loop_v16i8_to_v16i32(ptr %src, ptr %dst) { ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: Lloh2: ; CHECK-NEXT: adrp x8, lCPI24_0@PAGE -; CHECK-NEXT: movi.2d v0, #0xffffffffffffffff ; CHECK-NEXT: Lloh3: -; CHECK-NEXT: adrp x9, lCPI24_2@PAGE +; CHECK-NEXT: adrp x9, lCPI24_1@PAGE ; CHECK-NEXT: Lloh4: -; CHECK-NEXT: ldr q1, [x8, lCPI24_0@PAGEOFF] +; CHECK-NEXT: adrp x10, lCPI24_2@PAGE ; CHECK-NEXT: Lloh5: -; CHECK-NEXT: adrp x8, lCPI24_1@PAGE +; CHECK-NEXT: ldr q0, [x8, lCPI24_0@PAGEOFF] ; CHECK-NEXT: Lloh6: -; CHECK-NEXT: adrp x10, lCPI24_3@PAGE +; CHECK-NEXT: adrp x8, lCPI24_3@PAGE ; CHECK-NEXT: Lloh7: -; CHECK-NEXT: ldr q2, [x8, lCPI24_1@PAGEOFF] +; CHECK-NEXT: ldr q1, [x9, lCPI24_1@PAGEOFF] ; CHECK-NEXT: Lloh8: -; CHECK-NEXT: ldr q3, [x9, lCPI24_2@PAGEOFF] +; CHECK-NEXT: ldr q2, [x10, lCPI24_2@PAGEOFF] ; CHECK-NEXT: Lloh9: -; CHECK-NEXT: ldr q4, [x10, lCPI24_3@PAGEOFF] +; CHECK-NEXT: ldr q3, [x8, lCPI24_3@PAGEOFF] ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: LBB24_1: ; %loop ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: ldr q5, [x0, x8] +; CHECK-NEXT: ldr q4, [x0, x8] ; CHECK-NEXT: add x8, x8, #16 ; CHECK-NEXT: cmp x8, #128 -; CHECK-NEXT: cmgt.16b v6, v5, v0 -; CHECK-NEXT: tbl.16b v16, { v5 }, v1 -; CHECK-NEXT: tbl.16b v17, { v5 }, v2 -; CHECK-NEXT: tbl.16b v19, { v5 }, v3 -; CHECK-NEXT: tbl.16b v5, { v5 }, v4 -; CHECK-NEXT: sshll2.8h v7, v6, #0 -; CHECK-NEXT: sshll.8h v6, v6, #0 -; CHECK-NEXT: sshll2.4s v18, v7, #0 -; CHECK-NEXT: sshll.4s v7, v7, #0 -; CHECK-NEXT: sshll2.4s v20, v6, #0 +; CHECK-NEXT: cmge.16b v5, v4, #0 +; CHECK-NEXT: tbl.16b v7, { v4 }, v0 +; CHECK-NEXT: tbl.16b v16, { v4 }, v1 +; CHECK-NEXT: tbl.16b v18, { v4 }, v2 +; CHECK-NEXT: tbl.16b v4, { v4 }, v3 +; CHECK-NEXT: sshll2.8h v6, v5, #0 +; CHECK-NEXT: sshll.8h v5, v5, #0 +; CHECK-NEXT: sshll2.4s v17, v6, #0 ; CHECK-NEXT: sshll.4s v6, v6, #0 -; CHECK-NEXT: and.16b v16, v16, v18 -; CHECK-NEXT: and.16b v7, v17, v7 -; CHECK-NEXT: and.16b v17, v19, v20 -; CHECK-NEXT: and.16b v5, v5, v6 -; CHECK-NEXT: stp q7, q16, [x1, #32] -; CHECK-NEXT: stp q5, q17, [x1], #64 +; CHECK-NEXT: sshll2.4s v19, v5, #0 +; CHECK-NEXT: sshll.4s v5, v5, #0 +; CHECK-NEXT: and.16b v7, v7, v17 +; CHECK-NEXT: and.16b v6, v16, v6 +; CHECK-NEXT: and.16b v16, v18, v19 +; CHECK-NEXT: and.16b v4, v4, v5 +; CHECK-NEXT: stp q6, q7, [x1, #32] +; CHECK-NEXT: stp q4, q16, [x1], #64 ; CHECK-NEXT: b.ne LBB24_1 ; CHECK-NEXT: ; %bb.2: ; %exit ; CHECK-NEXT: ret ; CHECK-NEXT: .loh AdrpLdr Lloh6, Lloh9 -; CHECK-NEXT: .loh AdrpLdr Lloh5, Lloh7 -; CHECK-NEXT: .loh AdrpLdr Lloh3, Lloh8 -; CHECK-NEXT: .loh AdrpAdrp Lloh2, Lloh5 -; CHECK-NEXT: .loh AdrpLdr Lloh2, Lloh4 +; CHECK-NEXT: .loh AdrpLdr Lloh4, Lloh8 +; CHECK-NEXT: .loh AdrpLdr Lloh3, Lloh7 +; CHECK-NEXT: .loh AdrpAdrp Lloh2, Lloh6 +; CHECK-NEXT: .loh AdrpLdr Lloh2, Lloh5 entry: br label %loop @@ -645,52 +643,51 @@ define void @extension_in_loop_as_shuffle_v16i8_to_v16i32(ptr %src, ptr %dst) { ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: Lloh10: ; CHECK-NEXT: adrp x8, lCPI25_0@PAGE -; CHECK-NEXT: movi.2d v0, #0xffffffffffffffff ; CHECK-NEXT: Lloh11: -; CHECK-NEXT: adrp x9, lCPI25_2@PAGE +; CHECK-NEXT: adrp x9, lCPI25_1@PAGE ; CHECK-NEXT: Lloh12: -; CHECK-NEXT: ldr q1, [x8, lCPI25_0@PAGEOFF] +; CHECK-NEXT: adrp x10, lCPI25_2@PAGE ; CHECK-NEXT: Lloh13: -; CHECK-NEXT: adrp x8, lCPI25_1@PAGE +; CHECK-NEXT: ldr q0, [x8, lCPI25_0@PAGEOFF] ; CHECK-NEXT: Lloh14: -; CHECK-NEXT: adrp x10, lCPI25_3@PAGE +; CHECK-NEXT: adrp x8, lCPI25_3@PAGE ; CHECK-NEXT: Lloh15: -; CHECK-NEXT: ldr q2, [x8, lCPI25_1@PAGEOFF] +; CHECK-NEXT: ldr q1, [x9, lCPI25_1@PAGEOFF] ; CHECK-NEXT: Lloh16: -; CHECK-NEXT: ldr q3, [x9, lCPI25_2@PAGEOFF] +; CHECK-NEXT: ldr q2, [x10, lCPI25_2@PAGEOFF] ; CHECK-NEXT: Lloh17: -; CHECK-NEXT: ldr q4, [x10, lCPI25_3@PAGEOFF] +; CHECK-NEXT: ldr q3, [x8, lCPI25_3@PAGEOFF] ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: LBB25_1: ; %loop ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: ldr q5, [x0, x8] +; CHECK-NEXT: ldr q4, [x0, x8] ; CHECK-NEXT: add x8, x8, #16 ; CHECK-NEXT: cmp x8, #128 -; CHECK-NEXT: cmgt.16b v6, v5, v0 -; CHECK-NEXT: tbl.16b v16, { v5 }, v1 -; CHECK-NEXT: tbl.16b v17, { v5 }, v2 -; CHECK-NEXT: tbl.16b v19, { v5 }, v3 -; CHECK-NEXT: tbl.16b v5, { v5 }, v4 -; CHECK-NEXT: sshll2.8h v7, v6, #0 -; CHECK-NEXT: sshll.8h v6, v6, #0 -; CHECK-NEXT: sshll2.4s v18, v7, #0 -; CHECK-NEXT: sshll.4s v7, v7, #0 -; CHECK-NEXT: sshll2.4s v20, v6, #0 +; CHECK-NEXT: cmge.16b v5, v4, #0 +; CHECK-NEXT: tbl.16b v7, { v4 }, v0 +; CHECK-NEXT: tbl.16b v16, { v4 }, v1 +; CHECK-NEXT: tbl.16b v18, { v4 }, v2 +; CHECK-NEXT: tbl.16b v4, { v4 }, v3 +; CHECK-NEXT: sshll2.8h v6, v5, #0 +; CHECK-NEXT: sshll.8h v5, v5, #0 +; CHECK-NEXT: sshll2.4s v17, v6, #0 ; CHECK-NEXT: sshll.4s v6, v6, #0 -; CHECK-NEXT: and.16b v16, v16, v18 -; CHECK-NEXT: and.16b v7, v17, v7 -; CHECK-NEXT: and.16b v17, v19, v20 -; CHECK-NEXT: and.16b v5, v5, v6 -; CHECK-NEXT: stp q7, q16, [x1, #32] -; CHECK-NEXT: stp q5, q17, [x1], #64 +; CHECK-NEXT: sshll2.4s v19, v5, #0 +; CHECK-NEXT: sshll.4s v5, v5, #0 +; CHECK-NEXT: and.16b v7, v7, v17 +; CHECK-NEXT: and.16b v6, v16, v6 +; CHECK-NEXT: and.16b v16, v18, v19 +; CHECK-NEXT: and.16b v4, v4, v5 +; CHECK-NEXT: stp q6, q7, [x1, #32] +; CHECK-NEXT: stp q4, q16, [x1], #64 ; CHECK-NEXT: b.ne LBB25_1 ; CHECK-NEXT: ; %bb.2: ; %exit ; CHECK-NEXT: ret ; CHECK-NEXT: .loh AdrpLdr Lloh14, Lloh17 -; CHECK-NEXT: .loh AdrpLdr Lloh13, Lloh15 -; CHECK-NEXT: .loh AdrpLdr Lloh11, Lloh16 -; CHECK-NEXT: .loh AdrpAdrp Lloh10, Lloh13 -; CHECK-NEXT: .loh AdrpLdr Lloh10, Lloh12 +; CHECK-NEXT: .loh AdrpLdr Lloh12, Lloh16 +; CHECK-NEXT: .loh AdrpLdr Lloh11, Lloh15 +; CHECK-NEXT: .loh AdrpAdrp Lloh10, Lloh14 +; CHECK-NEXT: .loh AdrpLdr Lloh10, Lloh13 entry: br label %loop @@ -717,52 +714,51 @@ define void @shuffle_in_loop_is_no_extend_v16i8_to_v16i32(ptr %src, ptr %dst) { ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: Lloh18: ; CHECK-NEXT: adrp x8, lCPI26_0@PAGE -; CHECK-NEXT: movi.2d v0, #0xffffffffffffffff ; CHECK-NEXT: Lloh19: -; CHECK-NEXT: adrp x9, lCPI26_2@PAGE +; CHECK-NEXT: adrp x9, lCPI26_1@PAGE ; CHECK-NEXT: Lloh20: -; CHECK-NEXT: ldr q1, [x8, lCPI26_0@PAGEOFF] +; CHECK-NEXT: adrp x10, lCPI26_2@PAGE ; CHECK-NEXT: Lloh21: -; CHECK-NEXT: adrp x8, lCPI26_1@PAGE +; CHECK-NEXT: ldr q0, [x8, lCPI26_0@PAGEOFF] ; CHECK-NEXT: Lloh22: -; CHECK-NEXT: adrp x10, lCPI26_3@PAGE +; CHECK-NEXT: adrp x8, lCPI26_3@PAGE ; CHECK-NEXT: Lloh23: -; CHECK-NEXT: ldr q2, [x8, lCPI26_1@PAGEOFF] +; CHECK-NEXT: ldr q1, [x9, lCPI26_1@PAGEOFF] ; CHECK-NEXT: Lloh24: -; CHECK-NEXT: ldr q3, [x9, lCPI26_2@PAGEOFF] +; CHECK-NEXT: ldr q2, [x10, lCPI26_2@PAGEOFF] ; CHECK-NEXT: Lloh25: -; CHECK-NEXT: ldr q4, [x10, lCPI26_3@PAGEOFF] +; CHECK-NEXT: ldr q3, [x8, lCPI26_3@PAGEOFF] ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: LBB26_1: ; %loop ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: ldr q5, [x0, x8] +; CHECK-NEXT: ldr q4, [x0, x8] ; CHECK-NEXT: add x8, x8, #16 ; CHECK-NEXT: cmp x8, #128 -; CHECK-NEXT: cmgt.16b v6, v5, v0 -; CHECK-NEXT: tbl.16b v16, { v5 }, v1 -; CHECK-NEXT: tbl.16b v17, { v5 }, v2 -; CHECK-NEXT: tbl.16b v19, { v5 }, v3 -; CHECK-NEXT: tbl.16b v5, { v5 }, v4 -; CHECK-NEXT: sshll2.8h v7, v6, #0 -; CHECK-NEXT: sshll.8h v6, v6, #0 -; CHECK-NEXT: sshll2.4s v18, v7, #0 -; CHECK-NEXT: sshll.4s v7, v7, #0 -; CHECK-NEXT: sshll2.4s v20, v6, #0 +; CHECK-NEXT: cmge.16b v5, v4, #0 +; CHECK-NEXT: tbl.16b v7, { v4 }, v0 +; CHECK-NEXT: tbl.16b v16, { v4 }, v1 +; CHECK-NEXT: tbl.16b v18, { v4 }, v2 +; CHECK-NEXT: tbl.16b v4, { v4 }, v3 +; CHECK-NEXT: sshll2.8h v6, v5, #0 +; CHECK-NEXT: sshll.8h v5, v5, #0 +; CHECK-NEXT: sshll2.4s v17, v6, #0 ; CHECK-NEXT: sshll.4s v6, v6, #0 -; CHECK-NEXT: and.16b v16, v16, v18 -; CHECK-NEXT: and.16b v7, v17, v7 -; CHECK-NEXT: and.16b v17, v19, v20 -; CHECK-NEXT: and.16b v5, v5, v6 -; CHECK-NEXT: stp q7, q16, [x1, #32] -; CHECK-NEXT: stp q5, q17, [x1], #64 +; CHECK-NEXT: sshll2.4s v19, v5, #0 +; CHECK-NEXT: sshll.4s v5, v5, #0 +; CHECK-NEXT: and.16b v7, v7, v17 +; CHECK-NEXT: and.16b v6, v16, v6 +; CHECK-NEXT: and.16b v16, v18, v19 +; CHECK-NEXT: and.16b v4, v4, v5 +; CHECK-NEXT: stp q6, q7, [x1, #32] +; CHECK-NEXT: stp q4, q16, [x1], #64 ; CHECK-NEXT: b.ne LBB26_1 ; CHECK-NEXT: ; %bb.2: ; %exit ; CHECK-NEXT: ret ; CHECK-NEXT: .loh AdrpLdr Lloh22, Lloh25 -; CHECK-NEXT: .loh AdrpLdr Lloh21, Lloh23 -; CHECK-NEXT: .loh AdrpLdr Lloh19, Lloh24 -; CHECK-NEXT: .loh AdrpAdrp Lloh18, Lloh21 -; CHECK-NEXT: .loh AdrpLdr Lloh18, Lloh20 +; CHECK-NEXT: .loh AdrpLdr Lloh20, Lloh24 +; CHECK-NEXT: .loh AdrpLdr Lloh19, Lloh23 +; CHECK-NEXT: .loh AdrpAdrp Lloh18, Lloh22 +; CHECK-NEXT: .loh AdrpLdr Lloh18, Lloh21 entry: br label %loop diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-prefetch.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-prefetch.ll new file mode 100644 index 0000000000000000000000000000000000000000..b53610a0f22e50ab23ce5944049fe17f5cc18039 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-prefetch.ll @@ -0,0 +1,32 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -global-isel -mtriple=amdgcn -verify-machineinstrs -stop-after=irtranslator < %s | FileCheck %s + +define void @prefetch_read(ptr %ptr) { + ; CHECK-LABEL: name: prefetch_read + ; CHECK: bb.1 (%ir-block.0): + ; CHECK-NEXT: liveins: $vgpr0, $vgpr1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 + ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32) + ; CHECK-NEXT: G_PREFETCH [[MV]](p0), 0, 0, 0 :: (load unknown-size from %ir.ptr, align 1) + ; CHECK-NEXT: SI_RETURN + call void @llvm.prefetch.p0(ptr %ptr, i32 0, i32 0, i32 0) + ret void +} + +define void @prefetch_write(ptr %ptr) { + ; CHECK-LABEL: name: prefetch_write + ; CHECK: bb.1 (%ir-block.0): + ; CHECK-NEXT: liveins: $vgpr0, $vgpr1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 + ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32) + ; CHECK-NEXT: G_PREFETCH [[MV]](p0), 1, 1, 1 :: (store unknown-size into %ir.ptr, align 1) + ; CHECK-NEXT: SI_RETURN + call void @llvm.prefetch.p0(ptr %ptr, i32 1, i32 1, i32 1) + ret void +} + +declare void @llvm.prefetch.p0(ptr, i32, i32, i32) diff --git a/llvm/test/CodeGen/AMDGPU/codegen-internal-only-func.ll b/llvm/test/CodeGen/AMDGPU/codegen-internal-only-func.ll index 4760ddb65bf2d6e31ca69410d77e5b1516f5b20e..680fae1869600e1b61dadf2768e312bca66b1ca8 100644 --- a/llvm/test/CodeGen/AMDGPU/codegen-internal-only-func.ll +++ b/llvm/test/CodeGen/AMDGPU/codegen-internal-only-func.ll @@ -1,6 +1,7 @@ ; REQUIRES: asserts -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=OPT %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck -check-prefixes=OPT,COV4 %s ; RUN: not llc --crash -O0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=null %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck -check-prefixes=OPT,COV5 %s ; AMDGPUAttributor deletes the function "by accident" so it's never ; codegened with optimizations. @@ -14,9 +15,12 @@ ; OPT-NEXT: amdhsa.target: amdgcn-amd-amdhsa--gfx900 ; OPT-NEXT: amdhsa.version: ; OPT-NEXT: - 1 -; OPT-NEXT: - 1 -; OPT-NEXT: ... +; COV4: - 1 +; COV5: - 2 +; OPT: ... define internal i32 @func() { ret i32 0 } +!llvm.module.flags = !{!0} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/elf-header-osabi.ll b/llvm/test/CodeGen/AMDGPU/elf-header-osabi.ll index e5de7859dab6f9454f1e2eb72a9d9929b8d9bc20..f8fc3e1e764801b7659248992146c2f7e5c08848 100644 --- a/llvm/test/CodeGen/AMDGPU/elf-header-osabi.ll +++ b/llvm/test/CodeGen/AMDGPU/elf-header-osabi.ll @@ -1,9 +1,12 @@ ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=NONE %s ; RUN: llc -filetype=obj -mtriple=amdgcn-amd- -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=NONE %s ; RUN: llc -filetype=obj -mtriple=amdgcn-amd-unknown -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=NONE %s -; RUN: llc -filetype=obj -mtriple=amdgcn--amdhsa -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA %s -; RUN: llc -filetype=obj -mtriple=amdgcn-amd-amdhsa -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA %s -; RUN: llc -filetype=obj -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA %s +; RUN: llc -filetype=obj -mtriple=amdgcn--amdhsa -mcpu=gfx801 --amdhsa-code-object-version=4 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA,HSA4 %s +; RUN: llc -filetype=obj -mtriple=amdgcn-amd-amdhsa -mcpu=gfx801 --amdhsa-code-object-version=4 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA,HSA4 %s +; RUN: llc -filetype=obj -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx801 --amdhsa-code-object-version=4 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA,HSA4 %s +; RUN: llc -filetype=obj -mtriple=amdgcn--amdhsa -mcpu=gfx801 --amdhsa-code-object-version=5 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA,HSA5 %s +; RUN: llc -filetype=obj -mtriple=amdgcn-amd-amdhsa -mcpu=gfx801 --amdhsa-code-object-version=5 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA,HSA5 %s +; RUN: llc -filetype=obj -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx801 --amdhsa-code-object-version=5 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA,HSA5 %s ; RUN: llc -filetype=obj -mtriple=amdgcn--amdpal -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=PAL %s ; RUN: llc -filetype=obj -mtriple=amdgcn-amd-amdpal -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=PAL %s ; RUN: llc -filetype=obj -mtriple=amdgcn-unknown-amdpal -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=PAL %s @@ -13,7 +16,8 @@ ; NONE: OS/ABI: SystemV (0x0) ; HSA: OS/ABI: AMDGPU_HSA (0x40) -; HSA: ABIVersion: 2 +; HSA4: ABIVersion: 2 +; HSA5: ABIVersion: 3 ; PAL: OS/ABI: AMDGPU_PAL (0x41) ; PAL: ABIVersion: 0 ; MESA3D: OS/ABI: AMDGPU_MESA3D (0x42) diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll index 61017e809c86365848a87be4d77ae3f7a7d129ec..e2a3749c7c471d3379a5a3b4b67418e470fb3a25 100644 --- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll +++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll @@ -8,8 +8,8 @@ ; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx803 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s ; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx803 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s -; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX900,GFX9-SDAG,GFX900-SDAG %s -; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX900,GFX9-GISEL,GFX900-GISEL %s +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG,GFX900-SDAG,GFX900 %s +; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL,GFX900-GISEL,GFX900 %s ; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefixes=GFX9,GFX90A,GFX9-SDAG,GFX90A-SDAG %s ; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefixes=GFX9,GFX90A,GFX9-GISEL,GFX90A-GISEL %s @@ -17,6 +17,9 @@ ; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s ; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG %s +; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s + ; Test for integer mad formation for patterns used in clpeak define i32 @clpeak_imad_pat_i32(i32 %x, i32 %y) { @@ -143,6 +146,36 @@ define i32 @clpeak_imad_pat_i32(i32 %x, i32 %y) { ; GFX10-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_i32: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v0, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v5, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v2, v5, v[2:3] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v3, v2, v[3:4] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_i32: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v2, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add i32 %x, 1 %add = mul i32 %y18, %y @@ -271,6 +304,35 @@ define signext i16 @clpeak_imad_pat_i16(i16 signext %x, i16 signext %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 16 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v1, v0 +; GFX11-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v2, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v2, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %conv33 = add i16 %x, 1 %add = mul i16 %conv33, %y @@ -462,6 +524,38 @@ define <2 x i16> @clpeak_imad_pat_v2i16(<2 x i16> %x, <2 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v2i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v2, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v2i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <2 x i16> %x, %add = mul <2 x i16> %y18, %y @@ -719,6 +813,60 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v3i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v1, -1 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v4, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v5, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v5, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v3, v4, -1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v4, v1, -1 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v5, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v3i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v4, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v5, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v3, v5, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v5, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y48 = add <3 x i16> %x, %add = mul <3 x i16> %y48, %y @@ -1062,6 +1210,60 @@ define <4 x i16> @clpeak_imad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v4i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v1, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v4, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v5, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v5, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v3, v4, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v4, v1, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v5, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v4i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v4, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v5, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v3, v5, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v5, v1, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <4 x i16> %x, %add = mul <4 x i16> %y18, %y @@ -1184,6 +1386,35 @@ define zeroext i16 @clpeak_umad_pat_i16(i16 zeroext %x, i16 zeroext %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_umad_pat_i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v1, v0 +; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_umad_pat_i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v2, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v2, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %conv33 = add i16 %x, 1 %add = mul i16 %conv33, %y @@ -1375,6 +1606,38 @@ define <2 x i16> @clpeak_umad_pat_v2i16(<2 x i16> %x, <2 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_umad_pat_v2i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v2, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_umad_pat_v2i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <2 x i16> %x, %add = mul <2 x i16> %y18, %y @@ -1632,6 +1895,60 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_umad_pat_v3i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v1, -1 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v4, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v5, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v5, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v3, v4, -1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v4, v1, -1 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v5, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_umad_pat_v3i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v4, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v5, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v3, v5, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v5, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y48 = add <3 x i16> %x, %add = mul <3 x i16> %y48, %y @@ -1975,6 +2292,60 @@ define <4 x i16> @clpeak_umad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_umad_pat_v4i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v1, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v4, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v5, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v5, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v3, v4, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v4, v1, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v5, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_umad_pat_v4i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v4, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v5, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v3, v5, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v5, v1, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <4 x i16> %x, %add = mul <4 x i16> %y18, %y @@ -2186,6 +2557,54 @@ define <2 x i32> @clpeak_imad_pat_v2i32(<2 x i32> %x, <2 x i32> %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v2, v0 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v1, v3, v1 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v2i32: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v6, v0, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v6, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, v7, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[4:5], null, v2, v6, v[2:3] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[5:6], null, v3, v7, v[3:4] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v4, v2, v[4:5] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[1:2], null, v5, v3, v[5:6] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v2i32: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v4, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, v5, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v4 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, 1, v5 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v1, v3 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v2, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v3, v1 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <2 x i32> %x, %add = mul <2 x i32> %y18, %y @@ -2471,6 +2890,74 @@ define <3 x i32> @clpeak_imad_pat_v3i32(<3 x i32> %x, <3 x i32> %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u32 v1, v4, v1 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v2, v5, v2 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v3i32: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v2, 1, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v0, v3 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v8, v1, v4 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v9, v2, v5 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v7, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, v8, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v6, v9, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v3 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v1, v4 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v6, v5 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[5:6], null, v2, v7, v[2:3] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[6:7], null, v3, v8, v[3:4] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[7:8], null, v4, v9, v[4:5] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v5, v2, v[5:6] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[1:2], null, v6, v3, v[6:7] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[2:3], null, v7, v4, v[7:8] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v3i32: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v6, v0, v3 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v7, v1, v4 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v8, v2, v5 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v6, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, v7, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, v8, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v3 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v1, v4 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v2, v5 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, 1, v6 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v4, 1, v7 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v5, 1, v8 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v0, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v1, v4 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v2, v5 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v3, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v4, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v5, v2 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y48 = add <3 x i32> %x, %add = mul <3 x i32> %y48, %y @@ -2794,6 +3281,90 @@ define <4 x i32> @clpeak_imad_pat_v4i32(<4 x i32> %x, <4 x i32> %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u32 v2, v5, v2 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v3, v6, v3 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v4i32: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v2, 1, v2 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v3, 1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v8, v0, v4 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v9, v1, v5 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v10, v2, v6 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v11, v3, v7 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v8, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, v9, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v12, v10, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v4 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v11, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v1, v5 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v12, v6 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v0, v7 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[6:7], null, v2, v8, v[2:3] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[7:8], null, v3, v9, v[3:4] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[8:9], null, v4, v10, v[4:5] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[9:10], null, v5, v11, v[5:6] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v6, v2, v[6:7] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[1:2], null, v7, v3, v[7:8] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[2:3], null, v8, v4, v[8:9] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v9, v5, v[9:10] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v4i32: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v2 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, 1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v8, v0, v4 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v9, v1, v5 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v10, v2, v6 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v11, v3, v7 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v8, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, v9, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, v10, v2 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, v11, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v4, 1, v8 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v5, 1, v9 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v6, 1, v10 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v7, 1, v11 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v8, 1, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v1, v5 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v2, v6 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v6, v3, v7 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v2 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, 1, v3 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v8 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v4, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v5, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v6, v3 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <4 x i32> %x, %add = mul <4 x i32> %y18, %y @@ -2950,6 +3521,40 @@ define i32 @clpeak_imad_pat_i24(i32 %x, i32 %y) { ; GFX10-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_i24: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GFX11-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 24 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v1, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v5, v0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v2, v5, v[2:3] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v3, v2, v[3:4] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_i24: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 24 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v1, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v2, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %shl = shl i32 %x, 8 %shr = ashr exact i32 %shl, 8 @@ -3110,6 +3715,40 @@ define i32 @clpeak_imad_pat_u24(i32 %x, i32 %y) { ; GFX10-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_u24: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; GFX11-SDAG-NEXT: v_and_b32_e32 v1, 0xffffff, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v1, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v5, v0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v2, v5, v[2:3] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v3, v2, v[3:4] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_u24: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffffff, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v1, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v2, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %shl = and i32 %x, 16777215 %shl1 = and i32 %y, 16777215 @@ -3240,6 +3879,35 @@ define signext i8 @clpeak_imad_pat_i8(i8 signext %x, i8 signext %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_i8: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v1, v0 +; GFX11-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_i8: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v2, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v2, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %conv33 = add i8 %x, 1 %add = mul i8 %conv33, %y @@ -3447,6 +4115,60 @@ define <2 x i8> @clpeak_imad_pat_v2i8(<2 x i8> %x, <2 x i8> %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4 ; GFX10-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v2i8: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-SDAG-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mad_u16 v4, v1, v3, v1 +; GFX11-SDAG-NEXT: v_mad_u16 v5, v0, v2, v0 +; GFX11-SDAG-NEXT: v_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mul_lo_u16 v3, v4, v3 +; GFX11-SDAG-NEXT: v_mul_lo_u16 v2, v5, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v3, v1, v3 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v2, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v1, v3, v1 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v2, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_lshlrev_b16 v2, 8, v1 +; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0 +; GFX11-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_or_b32_e32 v0, v0, v2 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v2i8: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v4, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v4, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v5, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: v_add_nc_u16 v2, v4, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v3, v5, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_add_nc_u16 v4, v0, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v5, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <2 x i8> %x, %add = mul <2 x i8> %y18, %y @@ -3805,6 +4527,75 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u32 v3, v3, v4 ; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v2, v3 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_i64: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_co_u32 v4, vcc_lo, v0, 1 +; GFX11-SDAG-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v4, v3 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v4, v2, 0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v6, v5, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_add3_u32 v1, v1, v7, v6 +; GFX11-SDAG-NEXT: v_add_co_u32 v6, vcc_lo, v0, v4 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v1, v5, vcc_lo +; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v6, v3 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v6, v2, 0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v5, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v3, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_add3_u32 v4, v4, v7, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v4, v0 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[5:6], null, v3, v0, v[3:4] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add3_u32 v6, v2, v6, v1 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v5, v4 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v6, v3 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v5, v3, v[5:6] +; GFX11-SDAG-NEXT: v_add3_u32 v1, v4, v1, v2 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_i64: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v0, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v4, v2, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v6, v4, v3 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v7, v5, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v6, v7 +; GFX11-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v4 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v1, v5, vcc_lo +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v6, v2, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v6, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v7, v2 +; GFX11-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v4, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_add3_u32 v5, v5, v3, v2 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v4, v0, 0 +; GFX11-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v4, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v5, v0 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v5, vcc_lo +; GFX11-GISEL-NEXT: v_add3_u32 v3, v3, v1, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v2, v4, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v2, v5 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v3, v4 +; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v2, v3 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add i64 %x, 1 %add = mul i64 %y18, %y @@ -4458,6 +5249,120 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) { ; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v4, v7 ; GFX10-GISEL-NEXT: v_add3_u32 v3, v3, v6, v5 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v2i64: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_co_u32 v8, vcc_lo, v0, 1 +; GFX11-SDAG-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v1, vcc_lo +; GFX11-SDAG-NEXT: v_add_co_u32 v10, vcc_lo, v2, 1 +; GFX11-SDAG-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, 0, v3, vcc_lo +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v12, v9, v4 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v13, v8, v5 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v8, v4, 0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v14, v11, v6 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v15, v10, v7 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[2:3], null, v10, v6, 0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add3_u32 v1, v1, v13, v12 +; GFX11-SDAG-NEXT: v_add3_u32 v12, v3, v15, v14 +; GFX11-SDAG-NEXT: v_add_co_u32 v3, vcc_lo, v0, v8 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_add_co_ci_u32_e32 v8, vcc_lo, v1, v9, vcc_lo +; GFX11-SDAG-NEXT: v_add_co_u32 v9, vcc_lo, v2, v10 +; GFX11-SDAG-NEXT: v_add_co_ci_u32_e32 v10, vcc_lo, v12, v11, vcc_lo +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v11, v8, v4 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v13, v3, v5 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v14, v9, v7 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v10, v10, v6 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[7:8], null, v9, v6, 0 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[5:6], null, v3, v4, 0 +; GFX11-SDAG-NEXT: v_add3_u32 v8, v8, v14, v10 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add3_u32 v6, v6, v13, v11 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v11, v8, v2 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[9:10], null, v7, v2, v[7:8] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v13, v6, v0 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v5, v0, v[5:6] +; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v5, v1 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v7, v12 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v6, v3, v6 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_add3_u32 v4, v13, v4, v0 +; GFX11-SDAG-NEXT: v_add3_u32 v10, v11, v10, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v11, v4, v5 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v3, v5, v[3:4] +; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v9, v8 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v10, v7 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[2:3], null, v9, v7, v[9:10] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add3_u32 v1, v11, v1, v6 +; GFX11-SDAG-NEXT: v_add3_u32 v3, v5, v3, v4 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v2i64: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_co_u32 v8, vcc_lo, v0, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v1, vcc_lo +; GFX11-GISEL-NEXT: v_add_co_u32 v10, vcc_lo, v2, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, 0, v3, vcc_lo +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v8, v4, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v12, v8, v5 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v13, v9, v4 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v10, v6, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v14, v10, v7 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v15, v11, v6 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v12, v13 +; GFX11-GISEL-NEXT: v_add3_u32 v12, v3, v14, v15 +; GFX11-GISEL-NEXT: v_add_co_u32 v3, vcc_lo, v0, v8 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v13, vcc_lo, v1, v9, vcc_lo +; GFX11-GISEL-NEXT: v_add_co_u32 v10, vcc_lo, v2, v10 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, v12, v11, vcc_lo +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[8:9], null, v3, v4, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v3, v5 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v13, v13, v4 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[3:4], null, v10, v6, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v7, v10, v7 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v6, v11, v6 +; GFX11-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo +; GFX11-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, 1 +; GFX11-GISEL-NEXT: v_add3_u32 v9, v9, v5, v13 +; GFX11-GISEL-NEXT: v_add3_u32 v10, v4, v7, v6 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, 0, v12, vcc_lo +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v8, v0, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v8, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v9, v0 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[6:7], null, v3, v2, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v11, v3, v11 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v10, v2 +; GFX11-GISEL-NEXT: v_add_co_u32 v8, vcc_lo, v8, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v9, vcc_lo +; GFX11-GISEL-NEXT: v_add_co_u32 v12, vcc_lo, v3, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v10, vcc_lo, 0, v10, vcc_lo +; GFX11-GISEL-NEXT: v_add3_u32 v3, v5, v1, v0 +; GFX11-GISEL-NEXT: v_add3_u32 v5, v7, v11, v2 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v4, v8, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v4, v9 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v7, v3, v8 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v6, v12, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v6, v6, v10 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v5, v12 +; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v4, v7 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add3_u32 v3, v3, v6, v5 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <2 x i64> %x, %add = mul <2 x i64> %y18, %y @@ -4577,23 +5482,41 @@ define i32 @v_multi_use_mul_chain_add_other_use_all(i32 %arg, i32 %arg1, i32 %ar ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v5, v1 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; -; GFX900-LABEL: v_multi_use_mul_chain_add_other_use_all: -; GFX900: ; %bb.0: ; %bb -; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX900-NEXT: v_add_u32_e32 v0, 1, v0 -; GFX900-NEXT: v_mul_lo_u32 v2, v0, v1 -; GFX900-NEXT: v_add_u32_e32 v0, v2, v0 -; GFX900-NEXT: v_mul_lo_u32 v0, v0, v1 -; GFX900-NEXT: v_add_u32_e32 v1, 1, v2 -; GFX900-NEXT: v_mul_lo_u32 v5, v0, v1 -; GFX900-NEXT: global_store_dword v[3:4], v2, off -; GFX900-NEXT: s_waitcnt vmcnt(0) -; GFX900-NEXT: global_store_dword v[3:4], v0, off -; GFX900-NEXT: s_waitcnt vmcnt(0) -; GFX900-NEXT: global_store_dword v[3:4], v5, off -; GFX900-NEXT: s_waitcnt vmcnt(0) -; GFX900-NEXT: v_add_u32_e32 v0, v5, v1 -; GFX900-NEXT: s_setpc_b64 s[30:31] +; GFX900-SDAG-LABEL: v_multi_use_mul_chain_add_other_use_all: +; GFX900-SDAG: ; %bb.0: ; %bb +; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-SDAG-NEXT: v_add_u32_e32 v0, 1, v0 +; GFX900-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX900-SDAG-NEXT: v_add_u32_e32 v0, v2, v0 +; GFX900-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX900-SDAG-NEXT: v_add_u32_e32 v1, 1, v2 +; GFX900-SDAG-NEXT: v_mul_lo_u32 v5, v0, v1 +; GFX900-SDAG-NEXT: global_store_dword v[3:4], v2, off +; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) +; GFX900-SDAG-NEXT: global_store_dword v[3:4], v0, off +; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) +; GFX900-SDAG-NEXT: global_store_dword v[3:4], v5, off +; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) +; GFX900-SDAG-NEXT: v_add_u32_e32 v0, v5, v1 +; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX900-GISEL-LABEL: v_multi_use_mul_chain_add_other_use_all: +; GFX900-GISEL: ; %bb.0: ; %bb +; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-GISEL-NEXT: v_add_u32_e32 v0, 1, v0 +; GFX900-GISEL-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX900-GISEL-NEXT: v_add_u32_e32 v0, v2, v0 +; GFX900-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX900-GISEL-NEXT: v_add_u32_e32 v1, 1, v2 +; GFX900-GISEL-NEXT: v_mul_lo_u32 v5, v0, v1 +; GFX900-GISEL-NEXT: global_store_dword v[3:4], v2, off +; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) +; GFX900-GISEL-NEXT: global_store_dword v[3:4], v0, off +; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) +; GFX900-GISEL-NEXT: global_store_dword v[3:4], v5, off +; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) +; GFX900-GISEL-NEXT: v_add_u32_e32 v0, v5, v1 +; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31] ; ; GFX90A-SDAG-LABEL: v_multi_use_mul_chain_add_other_use_all: ; GFX90A-SDAG: ; %bb.0: ; %bb @@ -4652,6 +5575,26 @@ define i32 @v_multi_use_mul_chain_add_other_use_all(i32 %arg, i32 %arg1, i32 %ar ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX10-NEXT: v_add_nc_u32_e32 v0, v5, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: v_multi_use_mul_chain_add_other_use_all: +; GFX11: ; %bb.0: ; %bb +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-NEXT: v_add_nc_u32_e32 v0, v2, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX11-NEXT: v_add_nc_u32_e32 v0, 1, v2 +; GFX11-NEXT: v_mul_lo_u32 v5, v1, v0 +; GFX11-NEXT: global_store_b32 v[3:4], v2, off dlc +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_store_b32 v[3:4], v1, off dlc +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_store_b32 v[3:4], v5, off dlc +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_add_nc_u32_e32 v0, v5, v0 +; GFX11-NEXT: s_setpc_b64 s[30:31] bb: %i = add i32 %arg, 1 %i3 = mul i32 %i, %arg1 @@ -4761,21 +5704,37 @@ define i32 @v_multi_use_mul_chain_add_other_use_some(i32 %arg, i32 %arg1, i32 %a ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; -; GFX900-LABEL: v_multi_use_mul_chain_add_other_use_some: -; GFX900: ; %bb.0: ; %bb -; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX900-NEXT: v_add_u32_e32 v0, 1, v0 -; GFX900-NEXT: v_mul_lo_u32 v2, v0, v1 -; GFX900-NEXT: v_add_u32_e32 v0, v2, v0 -; GFX900-NEXT: v_mul_lo_u32 v0, v0, v1 -; GFX900-NEXT: v_add_u32_e32 v1, 1, v2 -; GFX900-NEXT: v_mul_lo_u32 v0, v0, v1 -; GFX900-NEXT: global_store_dword v[3:4], v2, off -; GFX900-NEXT: s_waitcnt vmcnt(0) -; GFX900-NEXT: global_store_dword v[3:4], v0, off -; GFX900-NEXT: s_waitcnt vmcnt(0) -; GFX900-NEXT: v_add_u32_e32 v0, v0, v1 -; GFX900-NEXT: s_setpc_b64 s[30:31] +; GFX900-SDAG-LABEL: v_multi_use_mul_chain_add_other_use_some: +; GFX900-SDAG: ; %bb.0: ; %bb +; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-SDAG-NEXT: v_add_u32_e32 v0, 1, v0 +; GFX900-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX900-SDAG-NEXT: v_add_u32_e32 v0, v2, v0 +; GFX900-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX900-SDAG-NEXT: v_add_u32_e32 v1, 1, v2 +; GFX900-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX900-SDAG-NEXT: global_store_dword v[3:4], v2, off +; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) +; GFX900-SDAG-NEXT: global_store_dword v[3:4], v0, off +; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) +; GFX900-SDAG-NEXT: v_add_u32_e32 v0, v0, v1 +; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX900-GISEL-LABEL: v_multi_use_mul_chain_add_other_use_some: +; GFX900-GISEL: ; %bb.0: ; %bb +; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-GISEL-NEXT: v_add_u32_e32 v0, 1, v0 +; GFX900-GISEL-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX900-GISEL-NEXT: v_add_u32_e32 v0, v2, v0 +; GFX900-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX900-GISEL-NEXT: v_add_u32_e32 v1, 1, v2 +; GFX900-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX900-GISEL-NEXT: global_store_dword v[3:4], v2, off +; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) +; GFX900-GISEL-NEXT: global_store_dword v[3:4], v0, off +; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) +; GFX900-GISEL-NEXT: v_add_u32_e32 v0, v0, v1 +; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31] ; ; GFX90A-SDAG-LABEL: v_multi_use_mul_chain_add_other_use_some: ; GFX90A-SDAG: ; %bb.0: ; %bb @@ -4828,6 +5787,24 @@ define i32 @v_multi_use_mul_chain_add_other_use_some(i32 %arg, i32 %arg1, i32 %a ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX10-NEXT: v_add_nc_u32_e32 v0, v5, v1 ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: v_multi_use_mul_chain_add_other_use_some: +; GFX11: ; %bb.0: ; %bb +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-NEXT: v_add_nc_u32_e32 v0, v2, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX11-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-NEXT: v_mul_lo_u32 v5, v0, v1 +; GFX11-NEXT: global_store_b32 v[3:4], v2, off dlc +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_store_b32 v[3:4], v5, off dlc +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_add_nc_u32_e32 v0, v5, v1 +; GFX11-NEXT: s_setpc_b64 s[30:31] bb: %i = add i32 %arg, 1 %i3 = mul i32 %i, %arg1 @@ -5045,6 +6022,58 @@ define i32 @clpeak_imad_pat_i32_x2(i32 %x, i32 %y) { ; GFX10-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_i32_x2: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v2, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, v2, v1 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v1, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v0, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, v5, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v2, v5, v[2:3] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v3, v2, v[3:4] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_i32_x2: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v2, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, v2, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, v2, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y38 = add i32 %x, 1 %add = mul i32 %y38, %y @@ -5424,6 +6453,98 @@ define <2 x i32> @clpeak_imad_pat_v2i32_x2(<2 x i32> %x, <2 x i32> %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v2, v0 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v1, v3, v1 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v2i32_x2: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v0, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v4, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, v5, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v2, 1, v4 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v3, 1, v5 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v0, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v2, v4, v2 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v3, v5, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v2, v0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v3, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v2, 1, v4 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v3, 1, v5 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v6, v0, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v2, v6, v2 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v3, v7, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v2, v0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v3, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[4:5], null, v2, v6, v[2:3] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[5:6], null, v3, v7, v[3:4] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v4, v2, v[4:5] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[1:2], null, v5, v3, v[5:6] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v2i32_x2: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v4, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, v5, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v4 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, 1, v5 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, v4, v2 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, v5, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v2, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v3, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v4 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, 1, v5 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, v4, v2 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, v5, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v2, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v3, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v4 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, 1, v5 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v1, v3 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v2, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v3, v1 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y38 = add <2 x i32> %x, %add = mul <2 x i32> %y38, %y @@ -5628,6 +6749,51 @@ define signext i16 @clpeak_imad_pat_i16_x2(i16 signext %x, i16 signext %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 16 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_i16_x2: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v1, v0 +; GFX11-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_i16_x2: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v2, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v2, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %conv69 = add i16 %x, 1 %add = mul i16 %conv69, %y @@ -5826,6 +6992,51 @@ define zeroext i16 @clpeak_umad_pat_i16_x2(i16 zeroext %x, i16 zeroext %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_umad_pat_i16_x2: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v1, v0 +; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_umad_pat_i16_x2: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v2, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v2, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %conv69 = add i16 %x, 1 %add = mul i16 %conv69, %y @@ -6149,6 +7360,60 @@ define <2 x i16> @clpeak_imad_pat_v2i16_x2(<2 x i16> %x, <2 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v2i16_x2: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v2, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_add_u16 v2, v1, v2 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v1, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v2, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v2, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v1, v0 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v2, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v2i16_x2: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v2, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v0, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v1, v2 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v2, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v2, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v1, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y38 = add <2 x i16> %x, %add = mul <2 x i16> %y38, %y @@ -6472,6 +7737,60 @@ define <2 x i16> @clpeak_umad_pat_v2i16_x2(<2 x i16> %x, <2 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_umad_pat_v2i16_x2: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v2, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_add_u16 v2, v1, v2 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v1, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v2, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v2, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v1, v0 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v2, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_umad_pat_v2i16_x2: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v2, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v0, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v1, v2 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v2, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v2, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v1, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y38 = add <2 x i16> %x, %add = mul <2 x i16> %y38, %y @@ -6525,6 +7844,15 @@ define <2 x i32> @multi_use_mul_mad_i32_var(i32 %x, i32 %y, i32 %z0, i32 %z1) { ; GFX10-NEXT: v_add_nc_u32_e32 v0, v1, v2 ; GFX10-NEXT: v_add_nc_u32_e32 v1, v1, v3 ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: multi_use_mul_mad_i32_var: +; GFX11: ; %bb.0: ; %entry +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_add_nc_u32_e32 v0, v1, v2 +; GFX11-NEXT: v_add_nc_u32_e32 v1, v1, v3 +; GFX11-NEXT: s_setpc_b64 s[30:31] entry: %mul = mul i32 %x, %y %add0 = add i32 %mul, %z0 @@ -6606,6 +7934,27 @@ define <2 x i16> @multi_use_mul_mad_i16_var(i16 %x, i16 %y, i16 %z0, i16 %z1) { ; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: multi_use_mul_mad_i16_var: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mad_u16 v2, v0, v1, v2 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_perm_b32 v0, v0, v2, 0x5040100 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: multi_use_mul_mad_i16_var: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v0, v2 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, v3 +; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %mul = mul i16 %x, %y %add0 = add i16 %mul, %z0 @@ -6653,6 +8002,16 @@ define i32 @other_use_mul_mad_i32_var(i32 %x, i32 %y, i32 %z, ptr addrspace(3) % ; GFX10-NEXT: ds_write_b32 v3, v1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: other_use_mul_mad_i32_var: +; GFX11: ; %bb.0: ; %entry +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_add_nc_u32_e32 v0, v1, v2 +; GFX11-NEXT: ds_store_b32 v3, v1 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_setpc_b64 s[30:31] entry: %mul = mul i32 %x, %y %add0 = add i32 %mul, %z @@ -6721,6 +8080,25 @@ define i16 @other_use_mul_mad_i16_var(i16 %x, i16 %y, i16 %z, ptr addrspace(3) % ; GFX10-GISEL-NEXT: ds_write_b16 v3, v1 ; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: other_use_mul_mad_i16_var: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mul_lo_u16 v4, v0, v1 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v1, v2 +; GFX11-SDAG-NEXT: ds_store_b16 v3, v4 +; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: other_use_mul_mad_i16_var: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v1, v2 +; GFX11-GISEL-NEXT: ds_store_b16 v3, v1 +; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %mul = mul i16 %x, %y %add0 = add i16 %mul, %z @@ -6816,6 +8194,15 @@ define <4 x i16> @multi_use_mul_mad_v2i16_var(<2 x i16> %x, <2 x i16> %y, <2 x i ; GFX10-NEXT: v_pk_add_u16 v0, v1, v2 ; GFX10-NEXT: v_pk_add_u16 v1, v1, v3 ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: multi_use_mul_mad_v2i16_var: +; GFX11: ; %bb.0: ; %entry +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_pk_mul_lo_u16 v1, v0, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_pk_add_u16 v0, v1, v2 +; GFX11-NEXT: v_pk_add_u16 v1, v1, v3 +; GFX11-NEXT: s_setpc_b64 s[30:31] entry: %mul = mul <2 x i16> %x, %y %add0 = add <2 x i16> %mul, %z0 @@ -6921,6 +8308,16 @@ define <2 x i16> @other_use_mul_mad_v2i16_var(<2 x i16> %x, <2 x i16> %y, <2 x i ; GFX10-NEXT: ds_write_b32 v3, v1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: other_use_mul_mad_v2i16_var: +; GFX11: ; %bb.0: ; %entry +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_pk_mul_lo_u16 v1, v0, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_pk_add_u16 v0, v1, v2 +; GFX11-NEXT: ds_store_b32 v3, v1 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_setpc_b64 s[30:31] entry: %mul = mul <2 x i16> %x, %y %add0 = add <2 x i16> %mul, %z @@ -6928,7 +8325,102 @@ entry: ret <2 x i16> %add0 } +define i64 @mul_u24_add64(i32 %x, i32 %y, i64 %z) { +; GFX67-LABEL: mul_u24_add64: +; GFX67: ; %bb.0: +; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX67-NEXT: v_mul_hi_u32_u24_e32 v4, v0, v1 +; GFX67-NEXT: v_mul_u32_u24_e32 v0, v0, v1 +; GFX67-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX67-NEXT: v_addc_u32_e32 v1, vcc, v4, v3, vcc +; GFX67-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-LABEL: mul_u24_add64: +; GFX8: ; %bb.0: +; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: v_mul_hi_u32_u24_e32 v4, v0, v1 +; GFX8-NEXT: v_mul_u32_u24_e32 v0, v0, v1 +; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2 +; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v4, v3, vcc +; GFX8-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-SDAG-LABEL: mul_u24_add64: +; GFX9-SDAG: ; %bb.0: +; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3] +; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-GISEL-LABEL: mul_u24_add64: +; GFX9-GISEL: ; %bb.0: +; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-GISEL-NEXT: v_mul_hi_u32_u24_e32 v4, v0, v1 +; GFX9-GISEL-NEXT: v_mul_u32_u24_e32 v0, v0, v1 +; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2 +; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v4, v3, vcc +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-SDAG-LABEL: mul_u24_add64: +; GFX10-SDAG: ; %bb.0: +; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v0, v1, v[2:3] +; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-GISEL-LABEL: mul_u24_add64: +; GFX10-GISEL: ; %bb.0: +; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-GISEL-NEXT: v_mul_u32_u24_e32 v4, v0, v1 +; GFX10-GISEL-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v1 +; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2 +; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo +; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] + %mul = call i64 @llvm.amdgcn.mul.u24.i64(i32 %x, i32 %y) + %add = add i64 %mul, %z + ret i64 %add +} + +define i64 @mul_u24_zext_add64(i32 %x, i32 %y, i64 %z) { +; GFX67-LABEL: mul_u24_zext_add64: +; GFX67: ; %bb.0: +; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX67-NEXT: v_mul_u32_u24_e32 v0, v0, v1 +; GFX67-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX67-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc +; GFX67-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-LABEL: mul_u24_zext_add64: +; GFX8: ; %bb.0: +; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: v_mul_u32_u24_e32 v0, v0, v1 +; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2 +; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc +; GFX8-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-LABEL: mul_u24_zext_add64: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_mul_u32_u24_e32 v0, v0, v1 +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-LABEL: mul_u24_zext_add64: +; GFX10: ; %bb.0: +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_mul_u32_u24_e32 v0, v0, v1 +; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2 +; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v3, vcc_lo +; GFX10-NEXT: s_setpc_b64 s[30:31] + %mul = call i32 @llvm.amdgcn.mul.u24(i32 %x, i32 %y) + %mul.zext = zext i32 %mul to i64 + %add = add i64 %mul.zext, %z + ret i64 %add +} + +declare i64 @llvm.amdgcn.mul.u24.i64(i32, i32) +declare i32 @llvm.amdgcn.mul.u24(i32, i32) + ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; GFX6: {{.*}} ; GFX7: {{.*}} +; GFX900: {{.*}} ; GFX90A: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll index d9c6fbb319019d330f8116f2801857400c5f0f21..cf588601016007fb0673ddc3aac0331f146083ee 100644 --- a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll +++ b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll @@ -444,31 +444,28 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no ; CHECK-NEXT: s_xor_b32 s4, exec_lo, s4 ; CHECK-NEXT: s_cbranch_execz .LBB0_31 ; CHECK-NEXT: ; %bb.30: ; in Loop: Header=BB0_28 Depth=1 -; CHECK-NEXT: v_xor_b32_e32 v5, v60, v58 -; CHECK-NEXT: v_lshrrev_b64 v[3:4], 16, v[56:57] -; CHECK-NEXT: v_mul_u32_u24_e32 v11, 0x180, v73 -; CHECK-NEXT: v_lshlrev_b32_e32 v0, 5, v0 -; CHECK-NEXT: v_lshrrev_b64 v[1:2], 16, v[45:46] -; CHECK-NEXT: v_lshlrev_b32_e32 v7, 16, v5 +; CHECK-NEXT: v_xor_b32_e32 v4, v60, v58 +; CHECK-NEXT: v_lshrrev_b64 v[2:3], 16, v[56:57] +; CHECK-NEXT: v_mad_u64_u32 v[6:7], null, 0x180, v73, s[46:47] +; CHECK-NEXT: v_lshlrev_b32_e32 v10, 5, v0 +; CHECK-NEXT: v_lshlrev_b32_e32 v1, 16, v4 ; CHECK-NEXT: v_lshlrev_b32_e32 v8, 6, v72 -; CHECK-NEXT: v_add_co_u32 v11, vcc_lo, s46, v11 -; CHECK-NEXT: v_lshlrev_b32_e32 v10, 12, v63 -; CHECK-NEXT: v_or_b32_e32 v4, v7, v4 -; CHECK-NEXT: v_mul_hi_u32_u24_e32 v7, 0x180, v73 -; CHECK-NEXT: v_xor_b32_e32 v6, v61, v59 -; CHECK-NEXT: v_lshlrev_b32_e32 v9, 16, v56 -; CHECK-NEXT: v_or3_b32 v10, v8, v10, v62 +; CHECK-NEXT: v_lshlrev_b32_e32 v9, 12, v63 +; CHECK-NEXT: v_xor_b32_e32 v5, v61, v59 +; CHECK-NEXT: v_lshlrev_b32_e32 v11, 16, v56 +; CHECK-NEXT: v_or_b32_e32 v3, v1, v3 +; CHECK-NEXT: v_lshrrev_b64 v[0:1], 16, v[45:46] +; CHECK-NEXT: v_add_co_u32 v6, vcc_lo, v6, v10 +; CHECK-NEXT: v_or3_b32 v8, v8, v9, v62 +; CHECK-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo +; CHECK-NEXT: v_lshrrev_b64 v[4:5], 16, v[4:5] +; CHECK-NEXT: v_or_b32_e32 v1, v11, v1 ; CHECK-NEXT: ; implicit-def: $vgpr42 ; CHECK-NEXT: ; implicit-def: $vgpr43 ; CHECK-NEXT: ; implicit-def: $vgpr44 -; CHECK-NEXT: v_add_co_ci_u32_e32 v12, vcc_lo, s47, v7, vcc_lo -; CHECK-NEXT: v_add_co_u32 v7, vcc_lo, v11, v0 -; CHECK-NEXT: v_lshrrev_b64 v[5:6], 16, v[5:6] -; CHECK-NEXT: v_add_co_ci_u32_e32 v8, vcc_lo, 0, v12, vcc_lo -; CHECK-NEXT: v_or_b32_e32 v2, v9, v2 -; CHECK-NEXT: global_store_dword v[7:8], v10, off offset:4 -; CHECK-NEXT: global_store_dwordx4 v[7:8], v[1:4], off offset:8 -; CHECK-NEXT: global_store_dwordx2 v[7:8], v[5:6], off offset:24 +; CHECK-NEXT: global_store_dword v[6:7], v8, off offset:4 +; CHECK-NEXT: global_store_dwordx4 v[6:7], v[0:3], off offset:8 +; CHECK-NEXT: global_store_dwordx2 v[6:7], v[4:5], off offset:24 ; CHECK-NEXT: .LBB0_31: ; %Flow ; CHECK-NEXT: ; in Loop: Header=BB0_28 Depth=1 ; CHECK-NEXT: s_andn2_saveexec_b32 s4, s4 diff --git a/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir b/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir index 83e85ccf7f8f19d234b6529d034abf7e557fadba..f0c5ba489ef3d5c3a5e778a43c20e2c68d94f37a 100644 --- a/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir +++ b/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir @@ -47,87 +47,46 @@ body: | name: live_through_test tracksRegLiveness: true body: | - ; RPU-LABEL: name: live_through_test - ; RPU: bb.0: - ; RPU-NEXT: Live-in: - ; RPU-NEXT: SGPR VGPR - ; RPU-NEXT: 0 0 - ; RPU-NEXT: 3 0 %0:sgpr_128 = IMPLICIT_DEF - ; RPU-NEXT: 3 0 - ; RPU-NEXT: Live-out: %0:00000000000000F3 - ; RPU-NEXT: Live-thr: - ; RPU-NEXT: 0 0 - ; RPU-NEXT: bb.1: - ; RPU-NEXT: Live-in: %0:00000000000000F3 - ; RPU-NEXT: SGPR VGPR - ; RPU-NEXT: 3 0 - ; RPU-NEXT: 3 0 S_NOP 0, implicit %0.sub0:sgpr_128 - ; RPU-NEXT: 2 0 - ; RPU-NEXT: 3 0 %0.sub0:sgpr_128 = IMPLICIT_DEF - ; RPU-NEXT: 3 0 - ; RPU-NEXT: 3 0 %0.sub1:sgpr_128 = IMPLICIT_DEF - ; RPU-NEXT: 3 0 - ; RPU-NEXT: 3 0 S_NOP 0, implicit %0.sub2:sgpr_128 - ; RPU-NEXT: 2 0 - ; RPU-NEXT: 3 0 %0.sub2:sgpr_128 = IMPLICIT_DEF - ; RPU-NEXT: 3 0 - ; RPU-NEXT: 3 0 S_NOP 0, implicit %0.sub2:sgpr_128 - ; RPU-NEXT: 2 0 - ; RPU-NEXT: 2 0 S_NOP 0, implicit %0.sub3:sgpr_128 - ; RPU-NEXT: 2 0 - ; RPU-NEXT: Live-out: %0:00000000000000C3 - ; RPU-NEXT: Live-thr: %0:00000000000000C0 - ; RPU-NEXT: 1 0 - ; RPU-NEXT: bb.2: - ; RPU-NEXT: Live-in: %0:00000000000000C3 - ; RPU-NEXT: SGPR VGPR - ; RPU-NEXT: 2 0 - ; RPU-NEXT: 2 0 S_NOP 0, implicit %0.sub3:sgpr_128, implicit %0.sub0:sgpr_128 - ; RPU-NEXT: 0 0 - ; RPU-NEXT: Live-out: - ; RPU-NEXT: Live-thr: - ; RPU-NEXT: 0 0 - ; - ; RPD-LABEL: name: live_through_test - ; RPD: bb.0: - ; RPD-NEXT: Live-in: - ; RPD-NEXT: SGPR VGPR - ; RPD-NEXT: 0 0 - ; RPD-NEXT: 4 0 %0:sgpr_128 = IMPLICIT_DEF - ; RPD-NEXT: 3 0 - ; RPD-NEXT: Live-out: %0:00000000000000F3 - ; RPD-NEXT: Live-thr: - ; RPD-NEXT: 0 0 - ; RPD-NEXT: bb.1: - ; RPD-NEXT: Live-in: %0:00000000000000F3 - ; RPD-NEXT: SGPR VGPR - ; RPD-NEXT: 3 0 - ; RPD-NEXT: 3 0 S_NOP 0, implicit %0.sub0:sgpr_128 - ; RPD-NEXT: 2 0 - ; RPD-NEXT: 3 0 %0.sub0:sgpr_128 = IMPLICIT_DEF - ; RPD-NEXT: 3 0 - ; RPD-NEXT: 4 0 %0.sub1:sgpr_128 = IMPLICIT_DEF - ; RPD-NEXT: 3 0 - ; RPD-NEXT: 3 0 S_NOP 0, implicit %0.sub2:sgpr_128 - ; RPD-NEXT: 2 0 - ; RPD-NEXT: 3 0 %0.sub2:sgpr_128 = IMPLICIT_DEF - ; RPD-NEXT: 3 0 - ; RPD-NEXT: 3 0 S_NOP 0, implicit %0.sub2:sgpr_128 - ; RPD-NEXT: 2 0 - ; RPD-NEXT: 2 0 S_NOP 0, implicit %0.sub3:sgpr_128 - ; RPD-NEXT: 2 0 - ; RPD-NEXT: Live-out: %0:00000000000000C3 - ; RPD-NEXT: Live-thr: %0:00000000000000C0 - ; RPD-NEXT: 1 0 - ; RPD-NEXT: bb.2: - ; RPD-NEXT: Live-in: %0:00000000000000C3 - ; RPD-NEXT: SGPR VGPR - ; RPD-NEXT: 2 0 - ; RPD-NEXT: 2 0 S_NOP 0, implicit %0.sub3:sgpr_128, implicit %0.sub0:sgpr_128 - ; RPD-NEXT: 0 0 - ; RPD-NEXT: Live-out: - ; RPD-NEXT: Live-thr: - ; RPD-NEXT: 0 0 + ; RP-LABEL: name: live_through_test + ; RP: bb.0: + ; RP-NEXT: Live-in: + ; RP-NEXT: SGPR VGPR + ; RP-NEXT: 0 0 + ; RP-NEXT: 4 0 %0:sgpr_128 = IMPLICIT_DEF + ; RP-NEXT: 3 0 + ; RP-NEXT: Live-out: %0:00000000000000F3 + ; RP-NEXT: Live-thr: + ; RP-NEXT: 0 0 + ; RP-NEXT: bb.1: + ; RP-NEXT: Live-in: %0:00000000000000F3 + ; RP-NEXT: SGPR VGPR + ; RP-NEXT: 3 0 + ; RP-NEXT: 3 0 S_NOP 0, implicit %0.sub0:sgpr_128 + ; RP-NEXT: 2 0 + ; RP-NEXT: 3 0 %0.sub0:sgpr_128 = IMPLICIT_DEF + ; RP-NEXT: 3 0 + ; RP-NEXT: 4 0 %0.sub1:sgpr_128 = IMPLICIT_DEF + ; RP-NEXT: 3 0 + ; RP-NEXT: 3 0 S_NOP 0, implicit %0.sub2:sgpr_128 + ; RP-NEXT: 2 0 + ; RP-NEXT: 3 0 %0.sub2:sgpr_128 = IMPLICIT_DEF + ; RP-NEXT: 3 0 + ; RP-NEXT: 3 0 S_NOP 0, implicit %0.sub2:sgpr_128 + ; RP-NEXT: 2 0 + ; RP-NEXT: 2 0 S_NOP 0, implicit %0.sub3:sgpr_128 + ; RP-NEXT: 2 0 + ; RP-NEXT: Live-out: %0:00000000000000C3 + ; RP-NEXT: Live-thr: %0:00000000000000C0 + ; RP-NEXT: 1 0 + ; RP-NEXT: bb.2: + ; RP-NEXT: Live-in: %0:00000000000000C3 + ; RP-NEXT: SGPR VGPR + ; RP-NEXT: 2 0 + ; RP-NEXT: 2 0 S_NOP 0, implicit %0.sub3:sgpr_128, implicit %0.sub0:sgpr_128 + ; RP-NEXT: 0 0 + ; RP-NEXT: Live-out: + ; RP-NEXT: Live-thr: + ; RP-NEXT: 0 0 bb.0: %0:sgpr_128 = IMPLICIT_DEF bb.1: @@ -223,7 +182,7 @@ body: | ; RPU-NEXT: 0 7 ; RPU-NEXT: 0 7 %7:vgpr_32 = GLOBAL_LOAD_DWORD %5:vreg_64, 0, 0, implicit $exec ; RPU-NEXT: 0 6 - ; RPU-NEXT: 0 7 %8:vreg_64 = IMPLICIT_DEF + ; RPU-NEXT: 0 8 %8:vreg_64 = IMPLICIT_DEF ; RPU-NEXT: 0 7 ; RPU-NEXT: 0 9 %9:vreg_64 = IMPLICIT_DEF ; RPU-NEXT: 0 9 @@ -262,7 +221,7 @@ body: | ; RPU-NEXT: 0 12 ; RPU-NEXT: 0 12 dead %21:vgpr_32 = GLOBAL_LOAD_DWORD %14:vreg_64, 0, 0, implicit $exec ; RPU-NEXT: 0 10 - ; RPU-NEXT: 0 10 dead %22:vgpr_32 = GLOBAL_LOAD_DWORD %15:vreg_64, 0, 0, implicit $exec + ; RPU-NEXT: 0 11 dead %22:vgpr_32 = GLOBAL_LOAD_DWORD %15:vreg_64, 0, 0, implicit $exec ; RPU-NEXT: 0 10 ; RPU-NEXT: 0 10 %23:vreg_64 = V_LSHLREV_B64_e64 2, %8:vreg_64, implicit $exec ; RPU-NEXT: 0 9 @@ -550,7 +509,7 @@ body: | ; RPU-NEXT: 0 0 ; RPU-NEXT: 0 0 $sgpr0 = S_BUFFER_LOAD_DWORD_IMM $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0 ; RPU-NEXT: 0 0 - ; RPU-NEXT: 0 0 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec + ; RPU-NEXT: 0 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec ; RPU-NEXT: 0 0 ; RPU-NEXT: 0 0 S_CMP_GT_U32 $sgpr0, 15, implicit-def $scc ; RPU-NEXT: 0 0 @@ -569,7 +528,7 @@ body: | ; RPU-NEXT: 0 1 ; RPU-NEXT: 0 1 $m0 = S_MOV_B32 killed $sgpr0 ; RPU-NEXT: 0 1 - ; RPU-NEXT: 0 1 %0:vreg_512 = V_INDIRECT_REG_WRITE_MOVREL_B32_V16 %0:vreg_512(tied-def 0), 42, 3, implicit $m0, implicit $exec + ; RPU-NEXT: 0 16 %0:vreg_512 = V_INDIRECT_REG_WRITE_MOVREL_B32_V16 %0:vreg_512(tied-def 0), 42, 3, implicit $m0, implicit $exec ; RPU-NEXT: 0 1 ; RPU-NEXT: Live-out: %0:0000000000000C00 ; RPU-NEXT: Live-thr: @@ -709,33 +668,19 @@ tracksRegLiveness: true body: | bb.0: liveins: $sgpr0_sgpr1_sgpr2_sgpr3 - ; RPU-LABEL: name: test_partially_used_early_clobber_def - ; RPU: Live-in: - ; RPU-NEXT: SGPR VGPR - ; RPU-NEXT: 0 0 - ; RPU-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3 - ; RPU-NEXT: 4 0 - ; RPU-NEXT: 5 0 early-clobber %1:sgpr_128 = COPY %0:sgpr_128 - ; RPU-NEXT: 1 0 - ; RPU-NEXT: 1 0 S_NOP 0, implicit %1.sub1:sgpr_128 - ; RPU-NEXT: 0 0 - ; RPU-NEXT: Live-out: - ; RPU-NEXT: Live-thr: - ; RPU-NEXT: 0 0 - ; - ; RPD-LABEL: name: test_partially_used_early_clobber_def - ; RPD: Live-in: - ; RPD-NEXT: SGPR VGPR - ; RPD-NEXT: 0 0 - ; RPD-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3 - ; RPD-NEXT: 4 0 - ; RPD-NEXT: 8 0 early-clobber %1:sgpr_128 = COPY %0:sgpr_128 - ; RPD-NEXT: 1 0 - ; RPD-NEXT: 1 0 S_NOP 0, implicit %1.sub1:sgpr_128 - ; RPD-NEXT: 0 0 - ; RPD-NEXT: Live-out: - ; RPD-NEXT: Live-thr: - ; RPD-NEXT: 0 0 + ; RP-LABEL: name: test_partially_used_early_clobber_def + ; RP: Live-in: + ; RP-NEXT: SGPR VGPR + ; RP-NEXT: 0 0 + ; RP-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3 + ; RP-NEXT: 4 0 + ; RP-NEXT: 8 0 early-clobber %1:sgpr_128 = COPY %0:sgpr_128 + ; RP-NEXT: 1 0 + ; RP-NEXT: 1 0 S_NOP 0, implicit %1.sub1:sgpr_128 + ; RP-NEXT: 0 0 + ; RP-NEXT: Live-out: + ; RP-NEXT: Live-thr: + ; RP-NEXT: 0 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3 early-clobber %1:sgpr_128 = COPY %0:sgpr_128 S_NOP 0, implicit %1.sub1 @@ -752,7 +697,7 @@ body: | ; RPU-NEXT: 0 0 ; RPU-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3 ; RPU-NEXT: 4 0 - ; RPU-NEXT: 7 0 %1:sgpr_128 = COPY %0:sgpr_128, implicit-def %2:sgpr_128, implicit-def early-clobber %3:sgpr_128, implicit-def dead early-clobber %4:sgpr_128 + ; RPU-NEXT: 16 0 %1:sgpr_128 = COPY %0:sgpr_128, implicit-def %2:sgpr_128, implicit-def early-clobber %3:sgpr_128, implicit-def dead early-clobber %4:sgpr_128 ; RPU-NEXT: 6 0 ; RPU-NEXT: 6 0 S_NOP 0, implicit %1.sub1:sgpr_128, implicit %2.sub0_sub1:sgpr_128, implicit %3.sub0_sub1_sub2:sgpr_128 ; RPU-NEXT: 0 0 diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll index 81346996431fc8669df6136fbef0ba8004d13c9f..b78f6412cd677ab8d9d8d0151b66462e4cdb4a68 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900" ; ASM: amdhsa.target: amdgcn-amd-amdhsa--gfx900 ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x12C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_ANY_V4 (0x100) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -30,4 +37,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll index de518bb99370adf37dde0e366f7ca389cea8c919..a3c75e0997533179c4168e631bcc56ad02853652 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx700" ; ASM: amdhsa.target: amdgcn-amd-amdhsa--gfx700 ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x22) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX700 (0x22) ; ELF-NEXT: ] @@ -29,4 +36,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll index b43d5941e8ccdacd0db05e540ffed9a64da4228d..d4d8af8c1a99be5d50de050292eb99f4416adce2 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack-" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack-' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x22C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_OFF_V4 (0x200) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -31,4 +38,4 @@ entry: attributes #0 = { "target-features"="-xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll index fc2bb4f4a28095a3ac8db976b4d7fd7c4af81921..9ca8b055a2797974db4cebaf5f7e1029df9c4bbb 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack+" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack+' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x32C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_ON_V4 (0x300) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -32,4 +39,4 @@ entry: attributes #0 = { "target-features"="+xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll index 27c58e400e132af3056ee46d263caff77e325e6c..fd3f5878469e66cd78bcebd9b7219c1db99f53b1 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack-" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack-' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x22C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_OFF_V4 (0x200) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -32,4 +39,4 @@ entry: attributes #0 = { "target-features"="-xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll index a97f8b2d9cd64528fe396accb124e72696ec1eef..34673dd5b891970ce2f5e119edd18c2def1c3120 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack-" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack-' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x22C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_OFF_V4 (0x200) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -32,4 +39,4 @@ entry: attributes #0 = { "target-features"="-xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll index 9ef7d7acd3c1421a06357bb9c7ef34ed713fd3e1..c283ece7e8bdffa8cd3ab3b89faa4a7b4ef7aaf8 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack+" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack+' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x32C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_ON_V4 (0x300) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -32,4 +39,4 @@ entry: attributes #0 = { "target-features"="+xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll index 5d32346745f840102397cf75df02b1d1609854b1..869254cae5258b0b6705f71b4f918782c2eaf91f 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack+" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack+' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x32C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_ON_V4 (0x300) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -32,4 +39,4 @@ entry: attributes #0 = { "target-features"="+xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll index d6778ec1f1fab2aac41cba0863cc70627bfeae76..b1bcb34c8aee48e173392cac053d7c6b4b5632b0 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900" ; ASM: amdhsa.target: amdgcn-amd-amdhsa--gfx900 ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x12C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_ANY_V4 (0x100) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -20,4 +27,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll index efa09a61e63053740732511331f393ada17d94c9..cc04eb0e661d659ac4c9543d1ff298fb1c366a35 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx700" ; ASM: amdhsa.target: amdgcn-amd-amdhsa--gfx700 ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x22) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX700 (0x22) ; ELF-NEXT: ] @@ -19,4 +26,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll index 630453656570c63819b6f7837ac0559ae4b50fa8..e84778b526f11d470fe4fad5a219b54dcc2beeac 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack-" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack-' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x22C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_OFF_V4 (0x200) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -22,4 +29,4 @@ entry: attributes #0 = { "target-features"="-xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll index f99ceccec96a97a41ed229c2c83178aa9bcb0ec3..a1ab6ed5f082acc64e33d75bc3fd94507b55e285 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack+" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack+' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x32C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_ON_V4 (0x300) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -22,4 +29,4 @@ entry: attributes #0 = { "target-features"="+xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/ARM/aapcs-hfa-code.ll b/llvm/test/CodeGen/ARM/aapcs-hfa-code.ll index e32f19ef6745280c35a10ca039aa19cf08560fbc..dabbb1e38a86bbac99ff10ae551e46d646ac099b 100644 --- a/llvm/test/CodeGen/ARM/aapcs-hfa-code.ll +++ b/llvm/test/CodeGen/ARM/aapcs-hfa-code.ll @@ -104,10 +104,8 @@ define arm_aapcs_vfpcc void @test_1double_nosplit([4 x float], [4 x double], [3 ; CHECK-NEXT: push {r11, lr} ; CHECK-NEXT: .pad #8 ; CHECK-NEXT: sub sp, sp, #8 -; CHECK-NEXT: movw r1, #0 -; CHECK-NEXT: mov r0, #0 -; CHECK-NEXT: movt r1, #16368 -; CHECK-NEXT: strd r0, r1, [sp] +; CHECK-NEXT: vmov.f64 d16, #1.000000e+00 +; CHECK-NEXT: vstr d16, [sp] ; CHECK-NEXT: bl test_1double_nosplit ; CHECK-NEXT: add sp, sp, #8 ; CHECK-NEXT: pop {r11, pc} @@ -138,10 +136,8 @@ define arm_aapcs_vfpcc void @test_1double_misaligned([4 x double], [4 x double], ; CHECK-NEXT: push {r11, lr} ; CHECK-NEXT: .pad #16 ; CHECK-NEXT: sub sp, sp, #16 -; CHECK-NEXT: movw r1, #0 -; CHECK-NEXT: mov r0, #0 -; CHECK-NEXT: movt r1, #16368 -; CHECK-NEXT: strd r0, r1, [sp, #8] +; CHECK-NEXT: vmov.f64 d16, #1.000000e+00 +; CHECK-NEXT: vstr d16, [sp, #8] ; CHECK-NEXT: bl test_1double_misaligned ; CHECK-NEXT: add sp, sp, #16 ; CHECK-NEXT: pop {r11, pc} diff --git a/llvm/test/CodeGen/ARM/ha-alignstack-call.ll b/llvm/test/CodeGen/ARM/ha-alignstack-call.ll index e861fe397f84944a07a8362931c60a5def417063..7e2a911c892816432644d46968aa86bef8604f1f 100644 --- a/llvm/test/CodeGen/ARM/ha-alignstack-call.ll +++ b/llvm/test/CodeGen/ARM/ha-alignstack-call.ll @@ -300,16 +300,16 @@ entry: ret double %call } ; CHECK-LABEL: g2_1_call: -; CHECK: movw r0, #0 -; CHECK: mov r1, #0 -; CHECK: movt r0, #16352 -; CHECK: str r1, [sp] -; CHECK: stmib sp, {r0, r1} -; CHECK: str r1, [sp, #12] -; CHECK: str r1, [sp, #16] -; CHECK: str r1, [sp, #20] -; CHECK: str r1, [sp, #24] -; CHECK: str r1, [sp, #28] +; CHECK: vmov.f64 d16, #5.000000e-01 +; CHECK: mov r0, #0 +; CHECK: str r0, [sp, #8] +; CHECK: str r0, [sp, #12] +; CHECK: str r0, [sp, #16] +; CHECK: vmov.i32 d0, #0x0 +; CHECK: str r0, [sp, #20] +; CHECK: str r0, [sp, #24] +; CHECK: str r0, [sp, #28] +; CHECK: vstr d16, [sp] ; CHECK: bl g2_1 ; pass in memory, alignment 8 diff --git a/llvm/test/CodeGen/ARM/vector-store.ll b/llvm/test/CodeGen/ARM/vector-store.ll index a8a1031637afc4d10544000ab6efd99f8f01662f..9c8ea7a2c440c833d3e8da56f967631eb2087f8b 100644 --- a/llvm/test/CodeGen/ARM/vector-store.ll +++ b/llvm/test/CodeGen/ARM/vector-store.ll @@ -403,17 +403,14 @@ define void @v3i8store(ptr %p) { ; CHECK-LABEL: v3i8store: ; CHECK: @ %bb.0: ; CHECK-NEXT: sub sp, #4 -; CHECK-NEXT: vmov.i32 d16, #0xff -; CHECK-NEXT: mov r1, sp -; CHECK-NEXT: vmov.i32 d17, #0x0 -; CHECK-NEXT: movs r2, #0 -; CHECK-NEXT: vand d16, d17, d16 -; CHECK-NEXT: vst1.32 {d16[0]}, [r1:32] -; CHECK-NEXT: vld1.32 {d16[0]}, [r1:32] +; CHECK-NEXT: movs r1, #0 +; CHECK-NEXT: mov r2, sp +; CHECK-NEXT: str r1, [sp] +; CHECK-NEXT: vld1.32 {d16[0]}, [r2:32] +; CHECK-NEXT: strb r1, [r0, #2] ; CHECK-NEXT: vmovl.u16 q8, d16 -; CHECK-NEXT: strb r2, [r0, #2] -; CHECK-NEXT: vmov.32 r1, d16[0] -; CHECK-NEXT: strh r1, [r0] +; CHECK-NEXT: vmov.32 r2, d16[0] +; CHECK-NEXT: strh r2, [r0] ; CHECK-NEXT: add sp, #4 ; CHECK-NEXT: bx lr store <3 x i8> zeroinitializer, ptr %p, align 4 diff --git a/llvm/test/CodeGen/BPF/GlobalISel/ir-translator-ret.ll b/llvm/test/CodeGen/BPF/GlobalISel/ir-translator-ret.ll new file mode 100644 index 0000000000000000000000000000000000000000..7a014f7841fc87655f2fbb4473189a448ee90325 --- /dev/null +++ b/llvm/test/CodeGen/BPF/GlobalISel/ir-translator-ret.ll @@ -0,0 +1,7 @@ +; RUN: llc -mtriple=bpfel -global-isel -verify-machineinstrs -stop-after=irtranslator < %s | FileCheck %s + +; CHECK: name: f +; CHECK: RET +define void @f() { + ret void +} diff --git a/llvm/test/CodeGen/DirectX/UAVMetadata.ll b/llvm/test/CodeGen/DirectX/UAVMetadata.ll index e86d53cd7afc83a79cb43239ec59fd5f0ee117dc..3d95723d6e49f0b8cc37d4b612f7c98065254759 100644 --- a/llvm/test/CodeGen/DirectX/UAVMetadata.ll +++ b/llvm/test/CodeGen/DirectX/UAVMetadata.ll @@ -37,16 +37,16 @@ target triple = "dxil-pc-shadermodel6.0-library" !hlsl.uavs = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9} -!0 = !{ptr @Zero, !"RWBuffer", i32 10, i32 0, i32 0} -!1 = !{ptr @One, !"Buffer>", i32 10, i32 1, i32 0} -!2 = !{ptr @Two, !"Buffer", i32 10, i32 2, i32 0} -!3 = !{ptr @Three, !"Buffer", i32 10, i32 3, i32 0} -!4 = !{ptr @Four, !"ByteAddressBuffer", i32 11, i32 5, i32 0} -!5 = !{ptr @Five, !"StructuredBuffer", i32 12, i32 6, i32 0} -!6 = !{ptr @Six, !"RasterizerOrderedBuffer", i32 10, i32 7, i32 0} -!7 = !{ptr @Seven, !"RasterizerOrderedStructuredBuffer", i32 12, i32 8, i32 0} -!8 = !{ptr @Eight, !"RasterizerOrderedByteAddressBuffer", i32 11, i32 9, i32 0} -!9 = !{ptr @Nine, !"RWBuffer", i32 10, i32 10, i32 2} +!0 = !{ptr @Zero, !"RWBuffer", i32 10, i1 false, i32 0, i32 0} +!1 = !{ptr @One, !"Buffer>", i32 10, i1 false, i32 1, i32 0} +!2 = !{ptr @Two, !"Buffer", i32 10, i1 false, i32 2, i32 0} +!3 = !{ptr @Three, !"Buffer", i32 10, i1 false, i32 3, i32 0} +!4 = !{ptr @Four, !"ByteAddressBuffer", i32 11, i1 false, i32 5, i32 0} +!5 = !{ptr @Five, !"StructuredBuffer", i32 12, i1 false, i32 6, i32 0} +!6 = !{ptr @Six, !"RasterizerOrderedBuffer", i32 10, i1 true, i32 7, i32 0} +!7 = !{ptr @Seven, !"RasterizerOrderedStructuredBuffer", i32 12, i1 true, i32 8, i32 0} +!8 = !{ptr @Eight, !"RasterizerOrderedByteAddressBuffer", i32 11, i1 true, i32 9, i32 0} +!9 = !{ptr @Nine, !"RWBuffer", i32 10, i1 false, i32 10, i32 2} ; CHECK: !dx.resources = !{[[ResList:[!][0-9]+]]} diff --git a/llvm/test/CodeGen/DirectX/cbuf.ll b/llvm/test/CodeGen/DirectX/cbuf.ll index 6640654f730e93b70775987d509b4353ee27610f..d07cc1e880b1a88828edebd68f49b4f974f054ec 100644 --- a/llvm/test/CodeGen/DirectX/cbuf.ll +++ b/llvm/test/CodeGen/DirectX/cbuf.ll @@ -34,4 +34,4 @@ attributes #1 = { nocallback nofree nosync nounwind readnone speculatable willre !hlsl.cbufs = !{!1} -!1 = !{ptr @A.cb., !"A.cb.ty", i32 13, i32 2, i32 1} +!1 = !{ptr @A.cb., !"A.cb.ty", i32 13, i1 false, i32 2, i32 1} diff --git a/llvm/test/CodeGen/DirectX/legacy_cb_layout_0.ll b/llvm/test/CodeGen/DirectX/legacy_cb_layout_0.ll index 50821467d5dd6a6f45c199306bd192f30b2448bf..0cfb839746b93edd1bb2a0b08ff844bd2e112e69 100644 --- a/llvm/test/CodeGen/DirectX/legacy_cb_layout_0.ll +++ b/llvm/test/CodeGen/DirectX/legacy_cb_layout_0.ll @@ -11,4 +11,4 @@ target triple = "dxil-unknown-shadermodel6.7-library" !hlsl.cbufs = !{!1} -!1 = !{ptr @A.cb., !"A.cb.ty", i32 13, i32 2, i32 0} +!1 = !{ptr @A.cb., !"A.cb.ty", i32 13, i1 false, i32 2, i32 0} diff --git a/llvm/test/CodeGen/DirectX/legacy_cb_layout_1.ll b/llvm/test/CodeGen/DirectX/legacy_cb_layout_1.ll index 8a38ccd8ef788b2b74c4bcd2fb61fdeb5390c578..b6d29f8d18d79f2344d40cb8ac93eec8ca0eafe1 100644 --- a/llvm/test/CodeGen/DirectX/legacy_cb_layout_1.ll +++ b/llvm/test/CodeGen/DirectX/legacy_cb_layout_1.ll @@ -34,4 +34,4 @@ target triple = "dxil-unknown-shadermodel6.7-library" !hlsl.cbufs = !{!0} -!0 = !{ptr @B.cb., !"B.cb.ty", i32 13, i32 1, i32 0} +!0 = !{ptr @B.cb., !"B.cb.ty", i32 13, i1 false, i32 1, i32 0} diff --git a/llvm/test/CodeGen/DirectX/legacy_cb_layout_2.ll b/llvm/test/CodeGen/DirectX/legacy_cb_layout_2.ll index b669538846ab1c540864a17ff54783b692c1fdac..d023d7906fdc52dd6ac8a9845a5d4e74beb9e4e4 100644 --- a/llvm/test/CodeGen/DirectX/legacy_cb_layout_2.ll +++ b/llvm/test/CodeGen/DirectX/legacy_cb_layout_2.ll @@ -47,5 +47,5 @@ target triple = "dxil-unknown-shadermodel6.7-library" !hlsl.cbufs = !{!0, !1} -!0 = !{ptr @B.cb., !"B.cb.ty", i32 13, i32 1, i32 0} -!1 = !{ptr @B.cb..1, !"B.cb.ty", i32 13, i32 2, i32 0} +!0 = !{ptr @B.cb., !"B.cb.ty", i32 13, i1 false, i32 1, i32 0} +!1 = !{ptr @B.cb..1, !"B.cb.ty", i32 13, i1 false, i32 2, i32 0} diff --git a/llvm/test/CodeGen/DirectX/legacy_cb_layout_3.ll b/llvm/test/CodeGen/DirectX/legacy_cb_layout_3.ll index afd46d08b5c712e2da2c33928b17c2dd8fffbd5f..38c2cd18b5ca1da641f4b81743d4df4aaeb35c7c 100644 --- a/llvm/test/CodeGen/DirectX/legacy_cb_layout_3.ll +++ b/llvm/test/CodeGen/DirectX/legacy_cb_layout_3.ll @@ -78,4 +78,4 @@ target triple = "dxil-unknown-shadermodel6.7-library" @D.cb. = external local_unnamed_addr constant { i32, %struct.B, half, %struct.C, double } !hlsl.cbufs = !{!0} -!0 = !{ptr @D.cb., !"D.cb.ty", i32 13, i32 1, i32 0} +!0 = !{ptr @D.cb., !"D.cb.ty", i32 13, i1 false, i32 1, i32 0} diff --git a/llvm/test/CodeGen/Generic/opt-codegen-no-target-machine.ll b/llvm/test/CodeGen/Generic/opt-codegen-no-target-machine.ll index 413f09ba55ddbfdc96ea2a4712df988730a31238..b7f0e7d02a21b542806dde816c063ca25288385b 100644 --- a/llvm/test/CodeGen/Generic/opt-codegen-no-target-machine.ll +++ b/llvm/test/CodeGen/Generic/opt-codegen-no-target-machine.ll @@ -1,3 +1,3 @@ -; RUN: not --crash opt %s -dwarfehprepare -o - 2>&1 | FileCheck %s +; RUN: not --crash opt %s -dwarf-eh-prepare -o - 2>&1 | FileCheck %s ; CHECK: Trying to construct TargetPassConfig without a target machine. Scheduling a CodeGen pass without a target triple set? diff --git a/llvm/test/CodeGen/LoongArch/lasx/fma-v4f64.ll b/llvm/test/CodeGen/LoongArch/lasx/fma-v4f64.ll new file mode 100644 index 0000000000000000000000000000000000000000..af18c52b096c801a846c741219942d5bdf2db3b8 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lasx/fma-v4f64.ll @@ -0,0 +1,804 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lasx --fp-contract=fast < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-FAST +; RUN: llc --mtriple=loongarch64 --mattr=+lasx --fp-contract=on < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-ON +; RUN: llc --mtriple=loongarch64 --mattr=+lasx --fp-contract=off < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-OFF + +define void @xvfmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfadd.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfadd.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul<4 x double> %v0, %v1 + %add = fadd<4 x double> %mul, %v2 + store <4 x double> %add, ptr %res + ret void +} + +define void @xvfmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul<4 x double> %v0, %v1 + %sub = fsub<4 x double> %mul, %v2 + store <4 x double> %sub, ptr %res + ret void +} + +define void @xvfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfadd.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfadd.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul<4 x double> %v0, %v1 + %add = fadd<4 x double> %mul, %v2 + %negadd = fneg<4 x double> %add + store <4 x double> %negadd, ptr %res + ret void +} + +define void @xvfnmadd_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmadd_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmadd_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr1, $xr1, 63 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmadd_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr1, $xr1, 63 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg nsz<4 x double> %v0 + %negv2 = fneg nsz<4 x double> %v2 + %mul = fmul nsz<4 x double> %negv0, %v1 + %add = fadd nsz<4 x double> %mul, %negv2 + store <4 x double> %add, ptr %res + ret void +} + +;; Check that xvfnmadd.d is not emitted. +define void @not_xvfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_xvfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_xvfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr1, $xr1, 63 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_xvfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr1, $xr1, 63 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg<4 x double> %v0 + %negv2 = fneg<4 x double> %v2 + %mul = fmul<4 x double> %negv0, %v1 + %add = fadd<4 x double> %mul, %negv2 + store <4 x double> %add, ptr %res + ret void +} + +define void @xvfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv2 = fneg<4 x double> %v2 + %mul = fmul<4 x double> %v0, %v1 + %add = fadd<4 x double> %mul, %negv2 + %neg = fneg<4 x double> %add + store <4 x double> %neg, ptr %res + ret void +} + +define void @xvfnmsub_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmsub_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmsub_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmsub_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg nsz<4 x double> %v0 + %mul = fmul nsz<4 x double> %negv0, %v1 + %add = fadd nsz<4 x double> %mul, %v2 + store <4 x double> %add, ptr %res + ret void +} + +;; Check that xvfnmsub.d is not emitted. +define void @not_xvfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_xvfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_xvfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_xvfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg<4 x double> %v0 + %mul = fmul<4 x double> %negv0, %v1 + %add = fadd<4 x double> %mul, %v2 + store <4 x double> %add, ptr %res + ret void +} + +define void @contract_xvfmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul contract <4 x double> %v0, %v1 + %add = fadd contract <4 x double> %mul, %v2 + store <4 x double> %add, ptr %res + ret void +} + +define void @contract_xvfmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul contract <4 x double> %v0, %v1 + %sub = fsub contract <4 x double> %mul, %v2 + store <4 x double> %sub, ptr %res + ret void +} + +define void @contract_xvfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul contract <4 x double> %v0, %v1 + %add = fadd contract <4 x double> %mul, %v2 + %negadd = fneg contract <4 x double> %add + store <4 x double> %negadd, ptr %res + ret void +} + +define void @contract_xvfnmadd_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmadd_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmadd_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmadd_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg contract nsz<4 x double> %v0 + %negv2 = fneg contract nsz<4 x double> %v2 + %mul = fmul contract nsz<4 x double> %negv0, %v1 + %add = fadd contract nsz<4 x double> %mul, %negv2 + store <4 x double> %add, ptr %res + ret void +} + +;; Check that xvfnmadd.d is not emitted. +define void @not_contract_xvfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_xvfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_xvfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-ON-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_xvfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-OFF-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg contract <4 x double> %v0 + %negv2 = fneg contract <4 x double> %v2 + %mul = fmul contract <4 x double> %negv0, %v1 + %add = fadd contract <4 x double> %mul, %negv2 + store <4 x double> %add, ptr %res + ret void +} + +define void @contract_xvfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv2 = fneg contract <4 x double> %v2 + %mul = fmul contract <4 x double> %v0, %v1 + %add = fadd contract <4 x double> %mul, %negv2 + %neg = fneg contract <4 x double> %add + store <4 x double> %neg, ptr %res + ret void +} + +define void @contract_xvfnmsub_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmsub_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmsub_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmsub_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg contract nsz<4 x double> %v0 + %mul = fmul contract nsz<4 x double> %negv0, %v1 + %add = fadd contract nsz<4 x double> %mul, %v2 + store <4 x double> %add, ptr %res + ret void +} + +;; Check that xvfnmsub.d is not emitted. +define void @not_contract_xvfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_xvfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_xvfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-ON-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_xvfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-OFF-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg contract <4 x double> %v0 + %mul = fmul contract <4 x double> %negv0, %v1 + %add = fadd contract <4 x double> %mul, %v2 + store <4 x double> %add, ptr %res + ret void +} + +define void @xvfmadd_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmadd_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmadd_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmadd_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul contract <4 x double> %v0, %v1 + %add = fadd contract <4 x double> %mul, %v2 + store <4 x double> %add, ptr %res + ret void +} + +define void @xvfmsub_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmsub_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmsub_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmsub_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul contract <4 x double> %v0, %v1 + %sub = fsub contract <4 x double> %mul, %v2 + store <4 x double> %sub, ptr %res + ret void +} + +define void @xvfnmadd_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmadd_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmadd_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmadd_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul contract <4 x double> %v0, %v1 + %add = fadd contract <4 x double> %mul, %v2 + %negadd = fneg contract <4 x double> %add + store <4 x double> %negadd, ptr %res + ret void +} + +define void @xvfnmsub_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmsub_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmsub_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmsub_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul contract <4 x double> %v0, %v1 + %negv2 = fneg contract <4 x double> %v2 + %add = fadd contract <4 x double> %negv2, %mul + %negadd = fneg contract <4 x double> %add + store <4 x double> %negadd, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lasx/fma-v8f32.ll b/llvm/test/CodeGen/LoongArch/lasx/fma-v8f32.ll new file mode 100644 index 0000000000000000000000000000000000000000..b7b3cb3a2e665ba654d6ee4bdd73df176f30f9cb --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lasx/fma-v8f32.ll @@ -0,0 +1,804 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lasx --fp-contract=fast < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-FAST +; RUN: llc --mtriple=loongarch64 --mattr=+lasx --fp-contract=on < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-ON +; RUN: llc --mtriple=loongarch64 --mattr=+lasx --fp-contract=off < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-OFF + +define void @xvfmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfadd.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfadd.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul<8 x float> %v0, %v1 + %add = fadd<8 x float> %mul, %v2 + store <8 x float> %add, ptr %res + ret void +} + +define void @xvfmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul<8 x float> %v0, %v1 + %sub = fsub<8 x float> %mul, %v2 + store <8 x float> %sub, ptr %res + ret void +} + +define void @xvfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfadd.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfadd.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul<8 x float> %v0, %v1 + %add = fadd<8 x float> %mul, %v2 + %negadd = fneg<8 x float> %add + store <8 x float> %negadd, ptr %res + ret void +} + +define void @xvfnmadd_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmadd_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmadd_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr1, $xr1, 31 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmadd_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr1, $xr1, 31 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg nsz<8 x float> %v0 + %negv2 = fneg nsz<8 x float> %v2 + %mul = fmul nsz<8 x float> %negv0, %v1 + %add = fadd nsz<8 x float> %mul, %negv2 + store <8 x float> %add, ptr %res + ret void +} + +;; Check that fnmadd.s is not emitted. +define void @not_xvfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_xvfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_xvfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr1, $xr1, 31 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_xvfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr1, $xr1, 31 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg<8 x float> %v0 + %negv2 = fneg<8 x float> %v2 + %mul = fmul<8 x float> %negv0, %v1 + %add = fadd<8 x float> %mul, %negv2 + store <8 x float> %add, ptr %res + ret void +} + +define void @xvfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv2 = fneg<8 x float> %v2 + %mul = fmul<8 x float> %v0, %v1 + %add = fadd<8 x float> %mul, %negv2 + %neg = fneg<8 x float> %add + store <8 x float> %neg, ptr %res + ret void +} + +define void @xvfnmsub_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmsub_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmsub_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmsub_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg nsz<8 x float> %v0 + %mul = fmul nsz<8 x float> %negv0, %v1 + %add = fadd nsz<8 x float> %mul, %v2 + store <8 x float> %add, ptr %res + ret void +} + +;; Check that fnmsub.s is not emitted. +define void @not_xvfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_xvfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_xvfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_xvfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg<8 x float> %v0 + %mul = fmul<8 x float> %negv0, %v1 + %add = fadd<8 x float> %mul, %v2 + store <8 x float> %add, ptr %res + ret void +} + +define void @contract_xvfmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul contract <8 x float> %v0, %v1 + %add = fadd contract <8 x float> %mul, %v2 + store <8 x float> %add, ptr %res + ret void +} + +define void @contract_xvfmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul contract <8 x float> %v0, %v1 + %sub = fsub contract <8 x float> %mul, %v2 + store <8 x float> %sub, ptr %res + ret void +} + +define void @contract_xvfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul contract <8 x float> %v0, %v1 + %add = fadd contract <8 x float> %mul, %v2 + %negadd = fneg contract <8 x float> %add + store <8 x float> %negadd, ptr %res + ret void +} + +define void @contract_xvfnmadd_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmadd_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmadd_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmadd_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg contract nsz<8 x float> %v0 + %negv2 = fneg contract nsz<8 x float> %v2 + %mul = fmul contract nsz<8 x float> %negv0, %v1 + %add = fadd contract nsz<8 x float> %mul, %negv2 + store <8 x float> %add, ptr %res + ret void +} + +;; Check that fnmadd.s is not emitted. +define void @not_contract_xvfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_xvfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_xvfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-ON-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_xvfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-OFF-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg contract <8 x float> %v0 + %negv2 = fneg contract <8 x float> %v2 + %mul = fmul contract <8 x float> %negv0, %v1 + %add = fadd contract <8 x float> %mul, %negv2 + store <8 x float> %add, ptr %res + ret void +} + +define void @contract_xvfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv2 = fneg contract <8 x float> %v2 + %mul = fmul contract <8 x float> %v0, %v1 + %add = fadd contract <8 x float> %mul, %negv2 + %neg = fneg contract <8 x float> %add + store <8 x float> %neg, ptr %res + ret void +} + +define void @contract_xvfnmsub_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmsub_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmsub_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmsub_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg contract nsz<8 x float> %v0 + %mul = fmul contract nsz<8 x float> %negv0, %v1 + %add = fadd contract nsz<8 x float> %mul, %v2 + store <8 x float> %add, ptr %res + ret void +} + +;; Check that fnmsub.s is not emitted. +define void @not_contract_xvfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_xvfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_xvfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-ON-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_xvfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-OFF-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg contract <8 x float> %v0 + %mul = fmul contract <8 x float> %negv0, %v1 + %add = fadd contract <8 x float> %mul, %v2 + store <8 x float> %add, ptr %res + ret void +} + +define void @xvfmadd_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmadd_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmadd_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmadd_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul contract <8 x float> %v0, %v1 + %add = fadd contract <8 x float> %mul, %v2 + store <8 x float> %add, ptr %res + ret void +} + +define void @xvfmsub_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmsub_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmsub_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmsub_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul contract <8 x float> %v0, %v1 + %sub = fsub contract <8 x float> %mul, %v2 + store <8 x float> %sub, ptr %res + ret void +} + +define void @xvfnmadd_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmadd_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmadd_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmadd_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul contract <8 x float> %v0, %v1 + %add = fadd contract <8 x float> %mul, %v2 + %negadd = fneg contract <8 x float> %add + store <8 x float> %negadd, ptr %res + ret void +} + +define void @xvfnmsub_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmsub_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmsub_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmsub_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul contract <8 x float> %v0, %v1 + %negv2 = fneg contract <8 x float> %v2 + %add = fadd contract <8 x float> %negv2, %mul + %negadd = fneg contract <8 x float> %add + store <8 x float> %negadd, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lasx/fsqrt.ll b/llvm/test/CodeGen/LoongArch/lasx/fsqrt.ll new file mode 100644 index 0000000000000000000000000000000000000000..c4a881bdeae9f1b81c756971141c75fb157b1d38 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lasx/fsqrt.ll @@ -0,0 +1,65 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s + +;; fsqrt +define void @sqrt_v8f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: sqrt_v8f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvfsqrt.s $xr0, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0, align 16 + %sqrt = call <8 x float> @llvm.sqrt.v8f32 (<8 x float> %v0) + store <8 x float> %sqrt, ptr %res, align 16 + ret void +} + +define void @sqrt_v4f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: sqrt_v4f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvfsqrt.d $xr0, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0, align 16 + %sqrt = call <4 x double> @llvm.sqrt.v4f64 (<4 x double> %v0) + store <4 x double> %sqrt, ptr %res, align 16 + ret void +} + +;; 1.0 / (fsqrt vec) +define void @one_div_sqrt_v8f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_div_sqrt_v8f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvfrsqrt.s $xr0, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0, align 16 + %sqrt = call <8 x float> @llvm.sqrt.v8f32 (<8 x float> %v0) + %div = fdiv <8 x float> , %sqrt + store <8 x float> %div, ptr %res, align 16 + ret void +} + +define void @one_div_sqrt_v4f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_div_sqrt_v4f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvfrsqrt.d $xr0, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0, align 16 + %sqrt = call <4 x double> @llvm.sqrt.v4f64 (<4 x double> %v0) + %div = fdiv <4 x double> , %sqrt + store <4 x double> %div, ptr %res, align 16 + ret void +} + +declare <8 x float> @llvm.sqrt.v8f32(<8 x float>) +declare <4 x double> @llvm.sqrt.v4f64(<4 x double>) diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fcmp.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fcmp.ll new file mode 100644 index 0000000000000000000000000000000000000000..ef67dbc100c04519e364fb9bf4e8d0b83c8ebb8c --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fcmp.ll @@ -0,0 +1,692 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s + +;; TREU +define void @v8f32_fcmp_true(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_true: +; CHECK: # %bb.0: +; CHECK-NEXT: xvrepli.b $xr0, -1 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp true <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +;; FALSE +define void @v4f64_fcmp_false(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_false: +; CHECK: # %bb.0: +; CHECK-NEXT: xvrepli.b $xr0, 0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp false <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETOEQ +define void @v8f32_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_oeq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.ceq.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp oeq <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_oeq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.ceq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp oeq <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETUEQ +define void @v8f32_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ueq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cueq.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp ueq <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ueq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cueq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp ueq <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETEQ +define void @v8f32_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.ceq.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp fast oeq <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.ceq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp fast ueq <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETOLE +define void @v8f32_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ole: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cle.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp ole <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ole: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp ole <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETULE +define void @v8f32_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cule.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp ule <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cule.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp ule <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETLE +define void @v8f32_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_le: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cle.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp fast ole <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_le: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp fast ule <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETOLT +define void @v8f32_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_olt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.clt.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp olt <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_olt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.clt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp olt <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETULT +define void @v8f32_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cult.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp ult <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cult.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp ult <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETLT +define void @v8f32_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_lt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.clt.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp fast olt <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_lt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.clt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp fast ult <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETONE +define void @v8f32_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_one: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cne.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp one <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_one: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cne.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp one <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETUNE +define void @v8f32_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_une: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cune.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp une <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_une: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cune.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp une <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETNE +define void @v8f32_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cne.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp fast one <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cne.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp fast une <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETO +define void @v8f32_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ord: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cor.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp ord <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ord: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cor.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp ord <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETUO +define void @v8f32_fcmp_uno(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_uno: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cun.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp uno <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_uno(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_uno: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cun.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp uno <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETOGT +define void @v8f32_fcmp_ogt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ogt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.clt.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp ogt <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ogt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ogt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.clt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp ogt <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGT +define void @v8f32_fcmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cult.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp ugt <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cult.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp ugt <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGT +define void @v8f32_fcmp_gt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_gt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.clt.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp fast ogt <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_gt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_gt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.clt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp fast ugt <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETOGE +define void @v8f32_fcmp_oge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_oge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cle.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp oge <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_oge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_oge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp oge <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGE +define void @v8f32_fcmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cule.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp uge <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cule.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp uge <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGE +define void @v8f32_fcmp_ge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cle.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp fast oge <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp fast uge <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fdiv.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fdiv.ll index 284121a79a492d03ba126d3d6748987f5f40cddc..6004565b0b784e5db52b6449d1febae0e9e24ea6 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fdiv.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fdiv.ll @@ -32,3 +32,32 @@ entry: store <4 x double> %v2, ptr %res ret void } + +;; 1.0 / vec +define void @one_fdiv_v8f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_fdiv_v8f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvfrecip.s $xr0, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %div = fdiv <8 x float> , %v0 + store <8 x float> %div, ptr %res + ret void +} + +define void @one_fdiv_v4f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_fdiv_v4f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvfrecip.d $xr0, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %div = fdiv <4 x double> , %v0 + store <4 x double> %div, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fneg.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fneg.ll new file mode 100644 index 0000000000000000000000000000000000000000..5eb468fc55a0ec6685ad9524d87615a1b6305bb4 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fneg.ll @@ -0,0 +1,29 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s + +define void @fneg_v8f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: fneg_v8f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = fneg <8 x float> %v0 + store <8 x float> %v1, ptr %res + ret void +} +define void @fneg_v4f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: fneg_v4f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = fneg <4 x double> %v0 + store <4 x double> %v1, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/icmp.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/icmp.ll new file mode 100644 index 0000000000000000000000000000000000000000..6693fe0f6ec7cec0b0a7bedea15611e4e644081b --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/icmp.ll @@ -0,0 +1,939 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s + +;; SETEQ +define void @v32i8_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v32i8_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvseqi.b $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %cmp = icmp eq <32 x i8> %v0, + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v32i8_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp eq <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i16_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvseqi.h $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %cmp = icmp eq <16 x i16> %v0, + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp eq <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i32_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvseqi.w $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %cmp = icmp eq <8 x i32> %v0, + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp eq <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i64_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvseqi.d $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %cmp = icmp eq <4 x i64> %v0, + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp eq <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETLE +define void @v32i8_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v32i8_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.b $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %cmp = icmp sle <32 x i8> %v0, + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v32i8_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp sle <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i16_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.h $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %cmp = icmp sle <16 x i16> %v0, + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp sle <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i32_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.w $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %cmp = icmp sle <8 x i32> %v0, + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp sle <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i64_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.d $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %cmp = icmp sle <4 x i64> %v0, + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp sle <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETULE +define void @v32i8_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v32i8_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.bu $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %cmp = icmp ule <32 x i8> %v0, + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v32i8_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.bu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp ule <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i16_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.hu $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %cmp = icmp ule <16 x i16> %v0, + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.hu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp ule <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i32_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.wu $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %cmp = icmp ule <8 x i32> %v0, + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.wu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp ule <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i64_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.du $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %cmp = icmp ule <4 x i64> %v0, + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.du $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp ule <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETLT +define void @v32i8_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v32i8_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.b $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %cmp = icmp slt <32 x i8> %v0, + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v32i8_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp slt <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i16_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.h $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %cmp = icmp slt <16 x i16> %v0, + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp slt <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i32_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.w $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %cmp = icmp slt <8 x i32> %v0, + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp slt <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i64_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.d $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %cmp = icmp slt <4 x i64> %v0, + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp slt <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETULT +define void @v32i8_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v32i8_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.bu $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %cmp = icmp ult <32 x i8> %v0, + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v32i8_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.bu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp ult <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i16_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.hu $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %cmp = icmp ult <16 x i16> %v0, + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.hu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp ult <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i32_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.wu $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %cmp = icmp ult <8 x i32> %v0, + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.wu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp ult <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i64_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.du $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %cmp = icmp ult <4 x i64> %v0, + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.du $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp ult <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETNE +define void @v32i8_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvxori.b $xr0, $xr0, 255 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp ne <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvrepli.b $xr1, -1 +; CHECK-NEXT: xvxor.v $xr0, $xr0, $xr1 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp ne <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvrepli.b $xr1, -1 +; CHECK-NEXT: xvxor.v $xr0, $xr0, $xr1 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp ne <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvrepli.b $xr1, -1 +; CHECK-NEXT: xvxor.v $xr0, $xr0, $xr1 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp ne <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGE +define void @v32i8_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp sge <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp sge <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp sge <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp sge <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGE +define void @v32i8_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.bu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp uge <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.hu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp uge <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.wu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp uge <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.du $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp uge <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGT +define void @v32i8_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp sgt <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp sgt <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp sgt <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp sgt <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGT +define void @v32i8_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.bu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp ugt <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.hu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp ugt <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.wu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp ugt <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.du $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp ugt <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lsx/fma-v2f64.ll b/llvm/test/CodeGen/LoongArch/lsx/fma-v2f64.ll new file mode 100644 index 0000000000000000000000000000000000000000..8e0459b4afabef3499b852d623365360231ebbb8 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lsx/fma-v2f64.ll @@ -0,0 +1,804 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lsx --fp-contract=fast < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-FAST +; RUN: llc --mtriple=loongarch64 --mattr=+lsx --fp-contract=on < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-ON +; RUN: llc --mtriple=loongarch64 --mattr=+lsx --fp-contract=off < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-OFF + +define void @vfmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfadd.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfadd.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul<2 x double> %v0, %v1 + %add = fadd<2 x double> %mul, %v2 + store <2 x double> %add, ptr %res + ret void +} + +define void @vfmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul<2 x double> %v0, %v1 + %sub = fsub<2 x double> %mul, %v2 + store <2 x double> %sub, ptr %res + ret void +} + +define void @vfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfadd.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfadd.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul<2 x double> %v0, %v1 + %add = fadd<2 x double> %mul, %v2 + %negadd = fneg<2 x double> %add + store <2 x double> %negadd, ptr %res + ret void +} + +define void @vfnmadd_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmadd_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmadd_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.d $vr1, $vr1, 63 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmadd_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr1, $vr1, 63 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg nsz<2 x double> %v0 + %negv2 = fneg nsz<2 x double> %v2 + %mul = fmul nsz<2 x double> %negv0, %v1 + %add = fadd nsz<2 x double> %mul, %negv2 + store <2 x double> %add, ptr %res + ret void +} + +;; Check that vfnmadd.d is not emitted. +define void @not_vfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_vfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_vfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.d $vr1, $vr1, 63 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_vfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr1, $vr1, 63 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg<2 x double> %v0 + %negv2 = fneg<2 x double> %v2 + %mul = fmul<2 x double> %negv0, %v1 + %add = fadd<2 x double> %mul, %negv2 + store <2 x double> %add, ptr %res + ret void +} + +define void @vfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv2 = fneg<2 x double> %v2 + %mul = fmul<2 x double> %v0, %v1 + %add = fadd<2 x double> %mul, %negv2 + %neg = fneg<2 x double> %add + store <2 x double> %neg, ptr %res + ret void +} + +define void @vfnmsub_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmsub_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmsub_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmsub_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg nsz<2 x double> %v0 + %mul = fmul nsz<2 x double> %negv0, %v1 + %add = fadd nsz<2 x double> %mul, %v2 + store <2 x double> %add, ptr %res + ret void +} + +;; Check that vfnmsub.d is not emitted. +define void @not_vfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_vfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_vfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_vfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg<2 x double> %v0 + %mul = fmul<2 x double> %negv0, %v1 + %add = fadd<2 x double> %mul, %v2 + store <2 x double> %add, ptr %res + ret void +} + +define void @contract_vfmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul contract <2 x double> %v0, %v1 + %add = fadd contract <2 x double> %mul, %v2 + store <2 x double> %add, ptr %res + ret void +} + +define void @contract_vfmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul contract <2 x double> %v0, %v1 + %sub = fsub contract <2 x double> %mul, %v2 + store <2 x double> %sub, ptr %res + ret void +} + +define void @contract_vfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul contract <2 x double> %v0, %v1 + %add = fadd contract <2 x double> %mul, %v2 + %negadd = fneg contract <2 x double> %add + store <2 x double> %negadd, ptr %res + ret void +} + +define void @contract_vfnmadd_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmadd_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmadd_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmadd_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg contract nsz<2 x double> %v0 + %negv2 = fneg contract nsz<2 x double> %v2 + %mul = fmul contract nsz<2 x double> %negv0, %v1 + %add = fadd contract nsz<2 x double> %mul, %negv2 + store <2 x double> %add, ptr %res + ret void +} + +;; Check that vfnmadd.d is not emitted. +define void @not_contract_vfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_vfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_vfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-ON-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_vfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-OFF-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg contract <2 x double> %v0 + %negv2 = fneg contract <2 x double> %v2 + %mul = fmul contract <2 x double> %negv0, %v1 + %add = fadd contract <2 x double> %mul, %negv2 + store <2 x double> %add, ptr %res + ret void +} + +define void @contract_vfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv2 = fneg contract <2 x double> %v2 + %mul = fmul contract <2 x double> %v0, %v1 + %add = fadd contract <2 x double> %mul, %negv2 + %neg = fneg contract <2 x double> %add + store <2 x double> %neg, ptr %res + ret void +} + +define void @contract_vfnmsub_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmsub_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmsub_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmsub_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg contract nsz<2 x double> %v0 + %mul = fmul contract nsz<2 x double> %negv0, %v1 + %add = fadd contract nsz<2 x double> %mul, %v2 + store <2 x double> %add, ptr %res + ret void +} + +;; Check that vfnmsub.d is not emitted. +define void @not_contract_vfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_vfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_vfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-ON-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_vfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-OFF-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg contract <2 x double> %v0 + %mul = fmul contract <2 x double> %negv0, %v1 + %add = fadd contract <2 x double> %mul, %v2 + store <2 x double> %add, ptr %res + ret void +} + +define void @vfmadd_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmadd_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmadd_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmadd_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul contract <2 x double> %v0, %v1 + %add = fadd contract <2 x double> %mul, %v2 + store <2 x double> %add, ptr %res + ret void +} + +define void @vfmsub_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmsub_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmsub_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmsub_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul contract <2 x double> %v0, %v1 + %sub = fsub contract <2 x double> %mul, %v2 + store <2 x double> %sub, ptr %res + ret void +} + +define void @vfnmadd_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmadd_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmadd_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmadd_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul contract <2 x double> %v0, %v1 + %add = fadd contract <2 x double> %mul, %v2 + %negadd = fneg contract <2 x double> %add + store <2 x double> %negadd, ptr %res + ret void +} + +define void @vfnmsub_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmsub_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmsub_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmsub_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul contract <2 x double> %v0, %v1 + %negv2 = fneg contract <2 x double> %v2 + %add = fadd contract <2 x double> %negv2, %mul + %negadd = fneg contract <2 x double> %add + store <2 x double> %negadd, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lsx/fma-v4f32.ll b/llvm/test/CodeGen/LoongArch/lsx/fma-v4f32.ll new file mode 100644 index 0000000000000000000000000000000000000000..7efbd61c0c4f7b5fac52ad297de76030b1835e6f --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lsx/fma-v4f32.ll @@ -0,0 +1,804 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lsx --fp-contract=fast < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-FAST +; RUN: llc --mtriple=loongarch64 --mattr=+lsx --fp-contract=on < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-ON +; RUN: llc --mtriple=loongarch64 --mattr=+lsx --fp-contract=off < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-OFF + +define void @vfmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfadd.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfadd.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul<4 x float> %v0, %v1 + %add = fadd<4 x float> %mul, %v2 + store <4 x float> %add, ptr %res + ret void +} + +define void @vfmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul<4 x float> %v0, %v1 + %sub = fsub<4 x float> %mul, %v2 + store <4 x float> %sub, ptr %res + ret void +} + +define void @vfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfadd.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfadd.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul<4 x float> %v0, %v1 + %add = fadd<4 x float> %mul, %v2 + %negadd = fneg<4 x float> %add + store <4 x float> %negadd, ptr %res + ret void +} + +define void @vfnmadd_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmadd_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmadd_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.w $vr1, $vr1, 31 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmadd_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr1, $vr1, 31 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg nsz<4 x float> %v0 + %negv2 = fneg nsz<4 x float> %v2 + %mul = fmul nsz<4 x float> %negv0, %v1 + %add = fadd nsz<4 x float> %mul, %negv2 + store <4 x float> %add, ptr %res + ret void +} + +;; Check that vfnmadd.s is not emitted. +define void @not_vfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_vfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_vfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.w $vr1, $vr1, 31 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_vfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr1, $vr1, 31 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg<4 x float> %v0 + %negv2 = fneg<4 x float> %v2 + %mul = fmul<4 x float> %negv0, %v1 + %add = fadd<4 x float> %mul, %negv2 + store <4 x float> %add, ptr %res + ret void +} + +define void @vfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv2 = fneg<4 x float> %v2 + %mul = fmul<4 x float> %v0, %v1 + %add = fadd<4 x float> %mul, %negv2 + %neg = fneg<4 x float> %add + store <4 x float> %neg, ptr %res + ret void +} + +define void @vfnmsub_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmsub_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmsub_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmsub_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg nsz<4 x float> %v0 + %mul = fmul nsz<4 x float> %negv0, %v1 + %add = fadd nsz<4 x float> %mul, %v2 + store <4 x float> %add, ptr %res + ret void +} + +;; Check that vfnmsub.s is not emitted. +define void @not_vfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_vfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_vfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_vfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg<4 x float> %v0 + %mul = fmul<4 x float> %negv0, %v1 + %add = fadd<4 x float> %mul, %v2 + store <4 x float> %add, ptr %res + ret void +} + +define void @contract_vfmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul contract <4 x float> %v0, %v1 + %add = fadd contract <4 x float> %mul, %v2 + store <4 x float> %add, ptr %res + ret void +} + +define void @contract_vfmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul contract <4 x float> %v0, %v1 + %sub = fsub contract <4 x float> %mul, %v2 + store <4 x float> %sub, ptr %res + ret void +} + +define void @contract_vfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul contract <4 x float> %v0, %v1 + %add = fadd contract <4 x float> %mul, %v2 + %negadd = fneg contract <4 x float> %add + store <4 x float> %negadd, ptr %res + ret void +} + +define void @contract_vfnmadd_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmadd_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmadd_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmadd_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg contract nsz<4 x float> %v0 + %negv2 = fneg contract nsz<4 x float> %v2 + %mul = fmul contract nsz<4 x float> %negv0, %v1 + %add = fadd contract nsz<4 x float> %mul, %negv2 + store <4 x float> %add, ptr %res + ret void +} + +;; Check that vfnmadd.s is not emitted. +define void @not_contract_vfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_vfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_vfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-ON-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_vfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-OFF-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg contract <4 x float> %v0 + %negv2 = fneg contract <4 x float> %v2 + %mul = fmul contract <4 x float> %negv0, %v1 + %add = fadd contract <4 x float> %mul, %negv2 + store <4 x float> %add, ptr %res + ret void +} + +define void @contract_vfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv2 = fneg contract <4 x float> %v2 + %mul = fmul contract <4 x float> %v0, %v1 + %add = fadd contract <4 x float> %mul, %negv2 + %neg = fneg contract <4 x float> %add + store <4 x float> %neg, ptr %res + ret void +} + +define void @contract_vfnmsub_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmsub_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmsub_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmsub_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg contract nsz<4 x float> %v0 + %mul = fmul contract nsz<4 x float> %negv0, %v1 + %add = fadd contract nsz<4 x float> %mul, %v2 + store <4 x float> %add, ptr %res + ret void +} + +;; Check that vfnmsub.s is not emitted. +define void @not_contract_vfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_vfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_vfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-ON-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_vfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-OFF-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg contract <4 x float> %v0 + %mul = fmul contract <4 x float> %negv0, %v1 + %add = fadd contract <4 x float> %mul, %v2 + store <4 x float> %add, ptr %res + ret void +} + +define void @vfmadd_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmadd_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmadd_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmadd_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul contract <4 x float> %v0, %v1 + %add = fadd contract <4 x float> %mul, %v2 + store <4 x float> %add, ptr %res + ret void +} + +define void @vfmsub_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmsub_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmsub_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmsub_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul contract <4 x float> %v0, %v1 + %sub = fsub contract <4 x float> %mul, %v2 + store <4 x float> %sub, ptr %res + ret void +} + +define void @vfnmadd_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmadd_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmadd_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmadd_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul contract <4 x float> %v0, %v1 + %add = fadd contract <4 x float> %mul, %v2 + %negadd = fneg contract <4 x float> %add + store <4 x float> %negadd, ptr %res + ret void +} + +define void @vfnmsub_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmsub_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmsub_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmsub_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul contract <4 x float> %v0, %v1 + %negv2 = fneg contract <4 x float> %v2 + %add = fadd contract <4 x float> %negv2, %mul + %negadd = fneg contract <4 x float> %add + store <4 x float> %negadd, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lsx/fsqrt.ll b/llvm/test/CodeGen/LoongArch/lsx/fsqrt.ll new file mode 100644 index 0000000000000000000000000000000000000000..a57bc1ca0e94885dd0a1a7e94a7748f5b38ee796 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lsx/fsqrt.ll @@ -0,0 +1,65 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lsx < %s | FileCheck %s + +;; fsqrt +define void @sqrt_v4f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: sqrt_v4f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vfsqrt.s $vr0, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0, align 16 + %sqrt = call <4 x float> @llvm.sqrt.v4f32 (<4 x float> %v0) + store <4 x float> %sqrt, ptr %res, align 16 + ret void +} + +define void @sqrt_v2f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: sqrt_v2f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vfsqrt.d $vr0, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0, align 16 + %sqrt = call <2 x double> @llvm.sqrt.v2f64 (<2 x double> %v0) + store <2 x double> %sqrt, ptr %res, align 16 + ret void +} + +;; 1.0 / (fsqrt vec) +define void @one_div_sqrt_v4f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_div_sqrt_v4f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vfrsqrt.s $vr0, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0, align 16 + %sqrt = call <4 x float> @llvm.sqrt.v4f32 (<4 x float> %v0) + %div = fdiv <4 x float> , %sqrt + store <4 x float> %div, ptr %res, align 16 + ret void +} + +define void @one_div_sqrt_v2f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_div_sqrt_v2f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vfrsqrt.d $vr0, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0, align 16 + %sqrt = call <2 x double> @llvm.sqrt.v2f64 (<2 x double> %v0) + %div = fdiv <2 x double> , %sqrt + store <2 x double> %div, ptr %res, align 16 + ret void +} + +declare <4 x float> @llvm.sqrt.v4f32(<4 x float>) +declare <2 x double> @llvm.sqrt.v2f64(<2 x double>) diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fcmp.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fcmp.ll new file mode 100644 index 0000000000000000000000000000000000000000..53fbf0b2f86fe43c363e29b321c5956782d3f536 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fcmp.ll @@ -0,0 +1,692 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lsx < %s | FileCheck %s + +;; TREU +define void @v4f32_fcmp_true(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_true: +; CHECK: # %bb.0: +; CHECK-NEXT: vrepli.b $vr0, -1 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp true <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +;; FALSE +define void @v2f64_fcmp_false(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_false: +; CHECK: # %bb.0: +; CHECK-NEXT: vrepli.b $vr0, 0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp false <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETOEQ +define void @v4f32_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_oeq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.ceq.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp oeq <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_oeq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.ceq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp oeq <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETUEQ +define void @v4f32_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ueq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cueq.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp ueq <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ueq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cueq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp ueq <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETEQ +define void @v4f32_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.ceq.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp fast oeq <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.ceq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp fast ueq <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETOLE +define void @v4f32_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ole: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cle.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp ole <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ole: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp ole <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETULE +define void @v4f32_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cule.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp ule <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cule.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp ule <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETLE +define void @v4f32_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_le: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cle.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp fast ole <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_le: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp fast ule <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETOLT +define void @v4f32_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_olt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.clt.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp olt <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_olt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.clt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp olt <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETULT +define void @v4f32_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cult.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp ult <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cult.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp ult <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETLT +define void @v4f32_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_lt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.clt.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp fast olt <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_lt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.clt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp fast ult <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETONE +define void @v4f32_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_one: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cne.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp one <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_one: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cne.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp one <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETUNE +define void @v4f32_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_une: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cune.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp une <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_une: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cune.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp une <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETNE +define void @v4f32_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cne.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp fast one <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cne.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp fast une <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETO +define void @v4f32_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ord: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cor.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp ord <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ord: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cor.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp ord <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETUO +define void @v4f32_fcmp_uno(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_uno: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cun.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp uno <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_uno(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_uno: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cun.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp uno <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETOGT +define void @v4f32_fcmp_ogt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ogt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.clt.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp ogt <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ogt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ogt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.clt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp ogt <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGT +define void @v4f32_fcmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cult.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp ugt <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cult.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp ugt <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGT +define void @v4f32_fcmp_gt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_gt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.clt.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp fast ogt <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_gt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_gt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.clt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp fast ugt <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETOGE +define void @v4f32_fcmp_oge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_oge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cle.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp oge <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_oge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_oge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp oge <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGE +define void @v4f32_fcmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cule.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp uge <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cule.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp uge <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGE +define void @v4f32_fcmp_ge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cle.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp fast oge <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp fast uge <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fdiv.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fdiv.ll index eb7c8bd9616ec7256531e737fab43c0a827940be..5f1ee9e4d212ebc5e68702d6928ef7d28f53064e 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fdiv.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fdiv.ll @@ -32,3 +32,32 @@ entry: store <2 x double> %v2, ptr %res ret void } + +;; 1.0 / vec +define void @one_fdiv_v4f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_fdiv_v4f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vfrecip.s $vr0, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %div = fdiv <4 x float> , %v0 + store <4 x float> %div, ptr %res + ret void +} + +define void @one_fdiv_v2f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_fdiv_v2f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vfrecip.d $vr0, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %div = fdiv <2 x double> , %v0 + store <2 x double> %div, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fneg.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fneg.ll new file mode 100644 index 0000000000000000000000000000000000000000..795c1ac8b368451c72dbe8f0bc8657d1335b64c0 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fneg.ll @@ -0,0 +1,29 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lsx < %s | FileCheck %s + +define void @fneg_v4f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: fneg_v4f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = fneg <4 x float> %v0 + store <4 x float> %v1, ptr %res + ret void +} +define void @fneg_v2f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: fneg_v2f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = fneg <2 x double> %v0 + store <2 x double> %v1, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/icmp.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/icmp.ll new file mode 100644 index 0000000000000000000000000000000000000000..448f3fa6c6e0e494b2b72cffee64fa5b2e787d16 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/icmp.ll @@ -0,0 +1,939 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lsx < %s | FileCheck %s + +;; SETEQ +define void @v16i8_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i8_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vseqi.b $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %cmp = icmp eq <16 x i8> %v0, + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v16i8_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp eq <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i16_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vseqi.h $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %cmp = icmp eq <8 x i16> %v0, + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp eq <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i32_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vseqi.w $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %cmp = icmp eq <4 x i32> %v0, + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp eq <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v2i64_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vseqi.d $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %cmp = icmp eq <2 x i64> %v0, + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp eq <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETLE +define void @v16i8_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i8_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.b $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %cmp = icmp sle <16 x i8> %v0, + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v16i8_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp sle <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i16_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.h $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %cmp = icmp sle <8 x i16> %v0, + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp sle <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i32_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.w $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %cmp = icmp sle <4 x i32> %v0, + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp sle <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v2i64_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.d $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %cmp = icmp sle <2 x i64> %v0, + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp sle <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETULE +define void @v16i8_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i8_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.bu $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %cmp = icmp ule <16 x i8> %v0, + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v16i8_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.bu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp ule <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i16_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.hu $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %cmp = icmp ule <8 x i16> %v0, + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.hu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp ule <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i32_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.wu $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %cmp = icmp ule <4 x i32> %v0, + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.wu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp ule <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v2i64_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.du $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %cmp = icmp ule <2 x i64> %v0, + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.du $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp ule <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETLT +define void @v16i8_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i8_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.b $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %cmp = icmp slt <16 x i8> %v0, + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v16i8_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp slt <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i16_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.h $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %cmp = icmp slt <8 x i16> %v0, + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp slt <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i32_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.w $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %cmp = icmp slt <4 x i32> %v0, + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp slt <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v2i64_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.d $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %cmp = icmp slt <2 x i64> %v0, + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp slt <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETULT +define void @v16i8_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i8_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.bu $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %cmp = icmp ult <16 x i8> %v0, + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v16i8_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.bu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp ult <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i16_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.hu $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %cmp = icmp ult <8 x i16> %v0, + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.hu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp ult <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i32_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.wu $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %cmp = icmp ult <4 x i32> %v0, + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.wu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp ult <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v2i64_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.du $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %cmp = icmp ult <2 x i64> %v0, + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.du $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp ult <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETNE +define void @v16i8_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vxori.b $vr0, $vr0, 255 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp ne <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vrepli.b $vr1, -1 +; CHECK-NEXT: vxor.v $vr0, $vr0, $vr1 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp ne <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vrepli.b $vr1, -1 +; CHECK-NEXT: vxor.v $vr0, $vr0, $vr1 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp ne <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vrepli.b $vr1, -1 +; CHECK-NEXT: vxor.v $vr0, $vr0, $vr1 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp ne <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGE +define void @v16i8_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp sge <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp sge <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp sge <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp sge <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGE +define void @v16i8_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.bu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp uge <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.hu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp uge <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.wu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp uge <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.du $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp uge <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGT +define void @v16i8_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp sgt <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp sgt <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp sgt <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp sgt <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGT +define void @v16i8_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.bu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp ugt <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.hu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp ugt <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.wu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp ugt <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.du $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp ugt <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/Mips/compactbranches/beqc-bnec-register-constraint.ll b/llvm/test/CodeGen/Mips/compactbranches/beqc-bnec-register-constraint.ll index 4c8674d46e29119fc4510af54fdf4b112c2b13c3..13e30676aa6fbcb107448f13499b60da99dc8b59 100644 --- a/llvm/test/CodeGen/Mips/compactbranches/beqc-bnec-register-constraint.ll +++ b/llvm/test/CodeGen/Mips/compactbranches/beqc-bnec-register-constraint.ll @@ -1,5 +1,5 @@ -; RUN: llc -march=mips -mcpu=mips32r6 -O1 -start-after=dwarfehprepare < %s | FileCheck %s -; RUN: llc -march=mips64 -mcpu=mips64r6 -O1 -start-after=dwarfehprepare < %s | FileCheck %s +; RUN: llc -march=mips -mcpu=mips32r6 -O1 -start-after=dwarf-eh-prepare < %s | FileCheck %s +; RUN: llc -march=mips64 -mcpu=mips64r6 -O1 -start-after=dwarf-eh-prepare < %s | FileCheck %s ; beqc/bnec have the constraint that $rs < $rt && $rs != 0 && $rt != 0 diff --git a/llvm/test/CodeGen/Mips/pr49200.ll b/llvm/test/CodeGen/Mips/pr49200.ll index 80a2bdd4e95ee3b5e9da1a360c83b0d4e4ba3a93..2a9f207b29e583b91d321443299b51c27445db4c 100644 --- a/llvm/test/CodeGen/Mips/pr49200.ll +++ b/llvm/test/CodeGen/Mips/pr49200.ll @@ -11,11 +11,10 @@ define dso_local void @foo() #0 { ; CHECK-LABEL: foo: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: addiusp -24 -; CHECK-NEXT: li16 $2, 0 -; CHECK-NEXT: sw $2, 4($sp) -; CHECK-NEXT: sw $2, 0($sp) -; CHECK-NEXT: sw $2, 12($sp) -; CHECK-NEXT: sw $2, 8($sp) +; CHECK-NEXT: mtc1 $zero, $f0 +; CHECK-NEXT: mthc1 $zero, $f0 +; CHECK-NEXT: sdc1 $f0, 0($sp) +; CHECK-NEXT: sdc1 $f0, 8($sp) ; CHECK-NEXT: ldc1 $f0, 0($sp) ; CHECK-NEXT: sdc1 $f0, 16($sp) ; CHECK-NEXT: addiusp 24 diff --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll index 5a6ab2926b40cfaefdeda289269ab17e51342446..a9faa130d6379f219ff6d07df035143cab736377 100644 --- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll +++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll @@ -227,3 +227,106 @@ define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 { %res = fpext <8 x bfloat> %load to <8 x float> ret <8 x float> %res } + +; CHECK-LABEL: test_fptosi_i16( +; CHECK: ld.param.b16 [[A:%rs[0-9]+]], [test_fptosi_i16_param_0]; +; SM80: cvt.f32.bf16 [[B:%f[0-9]+]], [[A]]; +; SM80: cvt.rzi.s16.f32 [[C:%rs[0-9]+]], [[B]]; +; SM80: cvt.u32.u16 [[R:%r[0-9]+]], [[C]]; +; SM90: cvt.rzi.s16.bf16 [[B:%rs[0-9]+]], [[A]]; +; SM90: cvt.u32.u16 [[R:%r[0-9]+]], [[B]]; +; CHECK: st.param.b32 [func_retval0+0], [[R]]; +; CHECK: ret; +define i16 @test_fptosi_i16(bfloat %a) { + %r = fptosi bfloat %a to i16 + ret i16 %r +} + +; CHECK-LABEL: test_fptoui_i16( +; CHECK: ld.param.b16 [[A:%rs[0-9]+]], [test_fptoui_i16_param_0]; +; SM80: cvt.f32.bf16 [[B:%f[0-9]+]], [[A]]; +; SM80: cvt.rzi.u16.f32 [[C:%rs[0-9]+]], [[B]]; +; SM80: cvt.u32.u16 [[R:%r[0-9]+]], [[C]]; +; SM90: cvt.rzi.u16.bf16 [[B:%rs[0-9]+]], [[A]]; +; SM90: cvt.u32.u16 [[R:%r[0-9]+]], [[B]]; +; CHECK: st.param.b32 [func_retval0+0], [[R]]; +; CHECK: ret; +define i16 @test_fptoui_i16(bfloat %a) { + %r = fptoui bfloat %a to i16 + ret i16 %r +} + +; CHECK-LABEL: test_sitofp_i16( +; CHECK: ld.param.u16 [[A:%rs[0-9]+]], [test_sitofp_i16_param_0]; +; SM80: cvt.rn.f32.s16 [[B:%f[0-9]+]], [[A]]; +; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; +; SM90: cvt.rn.bf16.s16 [[R:%rs[0-9]+]], [[A]]; +; CHECK: st.param.b16 [func_retval0+0], [[R]]; +; CHECK: ret; +define bfloat @test_sitofp_i16(i16 %a) { + %r = sitofp i16 %a to bfloat + ret bfloat %r +} + +; CHECK-LABEL: test_uitofp_i8( +; CHECK: ld.param.u8 %rs1, [test_uitofp_i8_param_0]; +; SM80: cvt.rn.f32.u16 [[B:%f[0-9]+]], [[A]]; +; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; +; SM90: cvt.rn.bf16.u16 [[R:%rs[0-9]+]], [[A]]; +; CHECK: st.param.b16 [func_retval0+0], [[R]]; +; CHECK: ret; +define bfloat @test_uitofp_i8(i8 %a) { + %r = uitofp i8 %a to bfloat + ret bfloat %r +} + +; CHECK-LABEL: test_uitofp_i1( +; CHECK: ld.param.u8 [[A:%rs[0-9]+]], [test_uitofp_i1_param_0]; +; CHECK: and.b16 [[B:%rs[0-9]+]], [[A]], 1; +; CHECK: setp.eq.b16 [[C:%p[0-9]+]], [[B]], 1; +; CHECK: selp.u32 [[D:%r[0-9]+]], 1, 0, [[C]]; +; SM80: cvt.rn.f32.u32 [[E:%f[0-9]+]], [[D]]; +; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[E]]; +; SM90: cvt.rn.bf16.u32 [[R:%rs[0-9]+]], [[D]]; +; CHECK: st.param.b16 [func_retval0+0], [[R]]; +; CHECK: ret; +define bfloat @test_uitofp_i1(i1 %a) { + %r = uitofp i1 %a to bfloat + ret bfloat %r +} + +; CHECK-LABEL: test_uitofp_i16( +; CHECK: ld.param.u16 [[A:%rs[0-9]+]], [test_uitofp_i16_param_0]; +; SM80: cvt.rn.f32.u16 [[B:%f[0-9]+]], [[A]]; +; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; +; SM90: cvt.rn.bf16.u16 [[R:%rs[0-9]+]], [[A]]; +; CHECK: st.param.b16 [func_retval0+0], [[R]]; +; CHECK: ret; +define bfloat @test_uitofp_i16(i16 %a) { + %r = uitofp i16 %a to bfloat + ret bfloat %r +} + +; CHECK-LABEL: test_uitofp_i32( +; CHECK: ld.param.u32 [[A:%r[0-9]+]], [test_uitofp_i32_param_0]; +; SM80: cvt.rn.f32.u32 [[B:%f[0-9]+]], [[A]]; +; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; +; SM90: cvt.rn.bf16.u32 [[R:%rs[0-9]+]], [[A]]; +; CHECK: st.param.b16 [func_retval0+0], [[R]]; +; CHECK: ret; +define bfloat @test_uitofp_i32(i32 %a) { + %r = uitofp i32 %a to bfloat + ret bfloat %r +} + +; CHECK-LABEL: test_uitofp_i64( +; CHECK: ld.param.u64 [[A:%rd[0-9]+]], [test_uitofp_i64_param_0]; +; SM80: cvt.rn.f32.u64 [[B:%f[0-9]+]], [[A]]; +; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; +; SM90: cvt.rn.bf16.u64 [[R:%rs[0-9]+]], [[A]]; +; CHECK: st.param.b16 [func_retval0+0], [[R]]; +; CHECK: ret; +define bfloat @test_uitofp_i64(i64 %a) { + %r = uitofp i64 %a to bfloat + ret bfloat %r +} diff --git a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll index 7fba1aadcd9861661ca448be362f5296a79095f9..c08f16c2644debb25bef1dcc0450a2373e531eb6 100644 --- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll +++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll @@ -392,6 +392,16 @@ define <2 x bfloat> @test_fabs(<2 x bfloat> %a) #0 { ret <2 x bfloat> %r } +; CHECK-LABEL: test_fabs_add( +; CHECK: abs.bf16x2 +; CHECK: ret; +define <2 x bfloat> @test_fabs_add(<2 x bfloat> %a, <2 x bfloat> %b) #0 { + %s = fadd <2 x bfloat> %a, %a + %r = call <2 x bfloat> @llvm.fabs.f16(<2 x bfloat> %s) + %d = fadd <2 x bfloat> %r, %b + ret <2 x bfloat> %d +} + ; CHECK-LABEL: test_minnum( ; CHECK-DAG: ld.param.b32 [[AF0:%r[0-9]+]], [test_minnum_param_0]; diff --git a/llvm/test/CodeGen/PowerPC/aix-ehinfo-sym.ll b/llvm/test/CodeGen/PowerPC/aix-ehinfo-sym.ll index dfbc871cbdb992a9eb0df90d6d658ec424b77a9d..06faf692cdca4fac448471beaa4f07d9f36a3e25 100644 --- a/llvm/test/CodeGen/PowerPC/aix-ehinfo-sym.ll +++ b/llvm/test/CodeGen/PowerPC/aix-ehinfo-sym.ll @@ -4,6 +4,10 @@ ; RUN: llc -mtriple powerpc64-ibm-aix -fast-isel -verify-machineinstrs < %s | \ ; RUN: FileCheck %s +; RUN: llc -mtriple powerpc64-ibm-aix -verify-machineinstrs -filetype=obj \ +; RUN: -o %t.o < %s +; RUN: llvm-readobj --syms %t.o | FileCheck --check-prefix=SYM %s + ; Function Attrs: nounwind declare i32 @func1() #0 @@ -47,4 +51,11 @@ attributes #0 = { nounwind } attributes #1 = { mustprogress noinline optnone } ; CHECK: __ehinfo.0: -; CHECK: .tc __ehinfo.0[TC],__ehinfo.0 +; CHECK: .tc __ehinfo.0[TE],__ehinfo.0 + +; SYM: Symbol { +; SYM: Name: __ehinfo.0 +; SYM: CSECT Auxiliary Entry { +; SYM: StorageMappingClass: XMC_TE (0x16) +; SYM: } +; SYM: } diff --git a/llvm/test/CodeGen/PowerPC/aix-emit-tracebacktable-clobber-register.ll b/llvm/test/CodeGen/PowerPC/aix-emit-tracebacktable-clobber-register.ll index 0cfe120f0ae42063b6580c32708a6bef8f5fe0e6..42bd4782c86887990e7166bcd2ce5b46bde9305e 100644 --- a/llvm/test/CodeGen/PowerPC/aix-emit-tracebacktable-clobber-register.ll +++ b/llvm/test/CodeGen/PowerPC/aix-emit-tracebacktable-clobber-register.ll @@ -104,7 +104,7 @@ entry: ; COMMON-NEXT: # -- End function ; COMMON: .toc ; COMMON: L..C2: -; COMMON-NEXT: .tc __ehinfo.1[TC],__ehinfo.1 +; COMMON-NEXT: .tc __ehinfo.1[TE],__ehinfo.1 ; OBJ-DIS: 9c: 00 00 00 00 # Traceback table start diff --git a/llvm/test/CodeGen/PowerPC/aix-exception.ll b/llvm/test/CodeGen/PowerPC/aix-exception.ll index eabf29b97d017ec19d9254b339d1127a3220ab07..5035d8e323d1589f543a3514c53a9201b9e5a017 100644 --- a/llvm/test/CodeGen/PowerPC/aix-exception.ll +++ b/llvm/test/CodeGen/PowerPC/aix-exception.ll @@ -180,7 +180,7 @@ eh.resume: ; preds = %catch.dispatch ; ASM: L..C0: ; ASM: .tc _ZTIi[TC],_ZTIi[UA] ; ASM: L..C1: -; ASM: .tc __ehinfo.1[TC],__ehinfo.1 +; ASM: .tc __ehinfo.1[TE],__ehinfo.1 declare ptr @__cxa_allocate_exception(i32) declare void @__cxa_throw(ptr, ptr, ptr) diff --git a/llvm/test/CodeGen/PowerPC/aix-tls-gd-double.ll b/llvm/test/CodeGen/PowerPC/aix-tls-gd-double.ll index 46ce3bfd450bd0d7e34cd2836b49bde1a8122efa..c0ffb8154c6917f34f30cfd868468becbae3d09e 100644 --- a/llvm/test/CodeGen/PowerPC/aix-tls-gd-double.ll +++ b/llvm/test/CodeGen/PowerPC/aix-tls-gd-double.ll @@ -24,7 +24,7 @@ define void @storesTGUninit(double %Val) #0 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C0(2) # target-flags(ppc-lo) @TGUninit +; SMALL32-NEXT: lwz 3, L..C0(2) # target-flags(ppc-tlsgdm) @TGUninit ; SMALL32-NEXT: lwz 4, L..C1(2) # target-flags(ppc-tlsgd) @TGUninit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -54,7 +54,7 @@ define void @storesTGUninit(double %Val) #0 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C0(2) # target-flags(ppc-lo) @TGUninit +; SMALL64-NEXT: ld 3, L..C0(2) # target-flags(ppc-tlsgdm) @TGUninit ; SMALL64-NEXT: ld 4, L..C1(2) # target-flags(ppc-tlsgd) @TGUninit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] @@ -90,7 +90,7 @@ define void @storesTGInit(double %Val) #0 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C2(2) # target-flags(ppc-lo) @TGInit +; SMALL32-NEXT: lwz 3, L..C2(2) # target-flags(ppc-tlsgdm) @TGInit ; SMALL32-NEXT: lwz 4, L..C3(2) # target-flags(ppc-tlsgd) @TGInit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -120,7 +120,7 @@ define void @storesTGInit(double %Val) #0 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C2(2) # target-flags(ppc-lo) @TGInit +; SMALL64-NEXT: ld 3, L..C2(2) # target-flags(ppc-tlsgdm) @TGInit ; SMALL64-NEXT: ld 4, L..C3(2) # target-flags(ppc-tlsgd) @TGInit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] @@ -156,7 +156,7 @@ define void @storesTIInit(double %Val) #0 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C4(2) # target-flags(ppc-lo) @TIInit +; SMALL32-NEXT: lwz 3, L..C4(2) # target-flags(ppc-tlsgdm) @TIInit ; SMALL32-NEXT: lwz 4, L..C5(2) # target-flags(ppc-tlsgd) @TIInit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -186,7 +186,7 @@ define void @storesTIInit(double %Val) #0 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C4(2) # target-flags(ppc-lo) @TIInit +; SMALL64-NEXT: ld 3, L..C4(2) # target-flags(ppc-tlsgdm) @TIInit ; SMALL64-NEXT: ld 4, L..C5(2) # target-flags(ppc-tlsgd) @TIInit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] @@ -222,7 +222,7 @@ define void @storesTWInit(double %Val) #0 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C6(2) # target-flags(ppc-lo) @TWInit +; SMALL32-NEXT: lwz 3, L..C6(2) # target-flags(ppc-tlsgdm) @TWInit ; SMALL32-NEXT: lwz 4, L..C7(2) # target-flags(ppc-tlsgd) @TWInit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -252,7 +252,7 @@ define void @storesTWInit(double %Val) #0 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C6(2) # target-flags(ppc-lo) @TWInit +; SMALL64-NEXT: ld 3, L..C6(2) # target-flags(ppc-tlsgdm) @TWInit ; SMALL64-NEXT: ld 4, L..C7(2) # target-flags(ppc-tlsgd) @TWInit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] @@ -288,7 +288,7 @@ define double @loadsTGUninit() #1 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C0(2) # target-flags(ppc-lo) @TGUninit +; SMALL32-NEXT: lwz 3, L..C0(2) # target-flags(ppc-tlsgdm) @TGUninit ; SMALL32-NEXT: lwz 4, L..C1(2) # target-flags(ppc-tlsgd) @TGUninit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -325,7 +325,7 @@ define double @loadsTGUninit() #1 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C0(2) # target-flags(ppc-lo) @TGUninit +; SMALL64-NEXT: ld 3, L..C0(2) # target-flags(ppc-tlsgdm) @TGUninit ; SMALL64-NEXT: ld 4, L..C1(2) # target-flags(ppc-tlsgd) @TGUninit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] @@ -370,7 +370,7 @@ define double @loadsTGInit() #1 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C2(2) # target-flags(ppc-lo) @TGInit +; SMALL32-NEXT: lwz 3, L..C2(2) # target-flags(ppc-tlsgdm) @TGInit ; SMALL32-NEXT: lwz 4, L..C3(2) # target-flags(ppc-tlsgd) @TGInit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -407,7 +407,7 @@ define double @loadsTGInit() #1 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C2(2) # target-flags(ppc-lo) @TGInit +; SMALL64-NEXT: ld 3, L..C2(2) # target-flags(ppc-tlsgdm) @TGInit ; SMALL64-NEXT: ld 4, L..C3(2) # target-flags(ppc-tlsgd) @TGInit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] @@ -452,7 +452,7 @@ define double @loadsTIInit() #1 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C4(2) # target-flags(ppc-lo) @TIInit +; SMALL32-NEXT: lwz 3, L..C4(2) # target-flags(ppc-tlsgdm) @TIInit ; SMALL32-NEXT: lwz 4, L..C5(2) # target-flags(ppc-tlsgd) @TIInit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -489,7 +489,7 @@ define double @loadsTIInit() #1 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C4(2) # target-flags(ppc-lo) @TIInit +; SMALL64-NEXT: ld 3, L..C4(2) # target-flags(ppc-tlsgdm) @TIInit ; SMALL64-NEXT: ld 4, L..C5(2) # target-flags(ppc-tlsgd) @TIInit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] @@ -534,7 +534,7 @@ define double @loadsTWInit() #1 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C6(2) # target-flags(ppc-lo) @TWInit +; SMALL32-NEXT: lwz 3, L..C6(2) # target-flags(ppc-tlsgdm) @TWInit ; SMALL32-NEXT: lwz 4, L..C7(2) # target-flags(ppc-tlsgd) @TWInit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -571,7 +571,7 @@ define double @loadsTWInit() #1 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C6(2) # target-flags(ppc-lo) @TWInit +; SMALL64-NEXT: ld 3, L..C6(2) # target-flags(ppc-tlsgdm) @TWInit ; SMALL64-NEXT: ld 4, L..C7(2) # target-flags(ppc-tlsgd) @TWInit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] diff --git a/llvm/test/CodeGen/PowerPC/aix-tls-gd-target-flags.ll b/llvm/test/CodeGen/PowerPC/aix-tls-gd-target-flags.ll index f52e213fee58fb0f65f704f62b28e279e42b0d68..f55932c3088de9be729fab0057fce5dc1d36d847 100644 --- a/llvm/test/CodeGen/PowerPC/aix-tls-gd-target-flags.ll +++ b/llvm/test/CodeGen/PowerPC/aix-tls-gd-target-flags.ll @@ -8,11 +8,11 @@ define signext i32 @foo() { ; CHECK-LABEL: name: foo ; CHECK: bb.0.entry: - ; CHECK-NEXT: [[LDtoc:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-lo) @a, $x2 :: (load (s64) from got) + ; CHECK-NEXT: [[LDtoc:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-tlsgdm) @a, $x2 :: (load (s64) from got) ; CHECK-NEXT: [[LDtoc1:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-tlsgd) @a, $x2 :: (load (s64) from got) ; CHECK-NEXT: [[TLSGDAIX8_:%[0-9]+]]:g8rc_and_g8rc_nox0 = TLSGDAIX8 killed [[LDtoc1]], killed [[LDtoc]] ; CHECK-NEXT: [[LWZ:%[0-9]+]]:gprc = LWZ 0, killed [[TLSGDAIX8_]] :: (dereferenceable load (s32) from @a) - ; CHECK-NEXT: [[LDtoc2:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-lo) @b, $x2 :: (load (s64) from got) + ; CHECK-NEXT: [[LDtoc2:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-tlsgdm) @b, $x2 :: (load (s64) from got) ; CHECK-NEXT: [[LDtoc3:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-tlsgd) @b, $x2 :: (load (s64) from got) ; CHECK-NEXT: [[TLSGDAIX8_1:%[0-9]+]]:g8rc_and_g8rc_nox0 = TLSGDAIX8 killed [[LDtoc3]], killed [[LDtoc2]] ; CHECK-NEXT: [[LWZ1:%[0-9]+]]:gprc = LWZ 0, killed [[TLSGDAIX8_1]] :: (dereferenceable load (s32) from @b) diff --git a/llvm/test/CodeGen/PowerPC/ctrloops-pseudo.ll b/llvm/test/CodeGen/PowerPC/ctrloops-pseudo.ll index 2fcc67087468d5fd408ccffcf7f161e580d96f51..e7c49c9dcc7d998d50beb1f79b2685ae797ecd16 100644 --- a/llvm/test/CodeGen/PowerPC/ctrloops-pseudo.ll +++ b/llvm/test/CodeGen/PowerPC/ctrloops-pseudo.ll @@ -375,7 +375,7 @@ define i32 @test4(i32 %inp) { ; AIX64-NEXT: B %bb.2 ; AIX64-NEXT: {{ $}} ; AIX64-NEXT: bb.2.return: - ; AIX64-NEXT: [[LDtoc:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-lo) @tls_var, $x2 :: (load (s64) from got) + ; AIX64-NEXT: [[LDtoc:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-tlsgdm) @tls_var, $x2 :: (load (s64) from got) ; AIX64-NEXT: [[LDtoc1:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-tlsgd) @tls_var, $x2 :: (load (s64) from got) ; AIX64-NEXT: [[TLSGDAIX8_:%[0-9]+]]:g8rc = TLSGDAIX8 killed [[LDtoc1]], killed [[LDtoc]] ; AIX64-NEXT: [[COPY2:%[0-9]+]]:gprc = COPY [[TLSGDAIX8_]].sub_32 @@ -405,7 +405,7 @@ define i32 @test4(i32 %inp) { ; AIX32-NEXT: B %bb.2 ; AIX32-NEXT: {{ $}} ; AIX32-NEXT: bb.2.return: - ; AIX32-NEXT: [[LWZtoc:%[0-9]+]]:gprc = LWZtoc target-flags(ppc-lo) @tls_var, $r2 :: (load (s32) from got) + ; AIX32-NEXT: [[LWZtoc:%[0-9]+]]:gprc = LWZtoc target-flags(ppc-tlsgdm) @tls_var, $r2 :: (load (s32) from got) ; AIX32-NEXT: [[LWZtoc1:%[0-9]+]]:gprc = LWZtoc target-flags(ppc-tlsgd) @tls_var, $r2 :: (load (s32) from got) ; AIX32-NEXT: [[TLSGDAIX:%[0-9]+]]:gprc = TLSGDAIX killed [[LWZtoc1]], killed [[LWZtoc]] ; AIX32-NEXT: [[ADD4_:%[0-9]+]]:gprc = ADD4 killed [[TLSGDAIX]], [[ISEL]] diff --git a/llvm/test/CodeGen/PowerPC/pcrel-tls-local-dynamic.ll b/llvm/test/CodeGen/PowerPC/pcrel-tls-local-dynamic.ll index 0b0fdbcaf0a4ea6bacaabee6ddb26ba11b866d67..da9844348f6033bf69268a8fd8cef5f683510016 100644 --- a/llvm/test/CodeGen/PowerPC/pcrel-tls-local-dynamic.ll +++ b/llvm/test/CodeGen/PowerPC/pcrel-tls-local-dynamic.ll @@ -13,7 +13,7 @@ define nonnull ptr @LocalDynamicAddressLoad() { ; CHECK-S-LABEL: LocalDynamicAddressLoad: ; CHECK-S: paddi r3, 0, x@got@tlsld@pcrel, 1 ; CHECK-S-NEXT: bl __tls_get_addr@notoc(x@tlsld) - ; CHECK-S-NEXT: paddi r3, r3, x@DTPREL, 0 + ; CHECK-S-NEXT: paddi r3, r3, x@DTPREL ; CHECK-S-NEXT: addi r1, r1, 32 ; CHECK-S-NEXT: ld r0, 16(r1) ; CHECK-S-NEXT: mtlr r0 @@ -34,7 +34,7 @@ define i32 @LocalDynamicValueLoad() { ; CHECK-S-LABEL: LocalDynamicValueLoad: ; CHECK-S: paddi r3, 0, x@got@tlsld@pcrel, 1 ; CHECK-S-NEXT: bl __tls_get_addr@notoc(x@tlsld) - ; CHECK-S-NEXT: paddi r3, r3, x@DTPREL, 0 + ; CHECK-S-NEXT: paddi r3, r3, x@DTPREL ; CHECK-S-NEXT: lwz r3, 0(r3) ; CHECK-S-NEXT: addi r1, r1, 32 ; CHECK-S-NEXT: ld r0, 16(r1) diff --git a/llvm/test/CodeGen/PowerPC/pcrel-tls-local-exec.ll b/llvm/test/CodeGen/PowerPC/pcrel-tls-local-exec.ll index 77d18e1b64de2fca41123596cc6dc7dacf3590cd..e4a22e0e2427e263d377aacec72f993647e68921 100644 --- a/llvm/test/CodeGen/PowerPC/pcrel-tls-local-exec.ll +++ b/llvm/test/CodeGen/PowerPC/pcrel-tls-local-exec.ll @@ -14,10 +14,10 @@ define dso_local ptr @LocalExecAddressLoad() { ; CHECK-S-LABEL: LocalExecAddressLoad: ; CHECK-S: # %bb.0: # %entry -; CHECK-S-NEXT: paddi r3, r13, x@TPREL, 0 +; CHECK-S-NEXT: paddi r3, r13, x@TPREL ; CHECK-S-NEXT: blr ; CHECK-O-LABEL: : -; CHECK-O: 0: paddi 3, 13, 0, 0 +; CHECK-O: 0: paddi 3, 13, 0 ; CHECK-O-NEXT: 0000000000000000: R_PPC64_TPREL34 x ; CHECK-O-NEXT: 8: blr entry: @@ -27,11 +27,11 @@ entry: define dso_local i32 @LocalExecValueLoad() { ; CHECK-S-LABEL: LocalExecValueLoad: ; CHECK-S: # %bb.0: # %entry -; CHECK-S-NEXT: paddi r3, r13, x@TPREL, 0 +; CHECK-S-NEXT: paddi r3, r13, x@TPREL ; CHECK-S-NEXT: lwz r3, 0(r3) ; CHECK-S-NEXT: blr ; CHECK-O-LABEL: : -; CHECK-O: 20: paddi 3, 13, 0, 0 +; CHECK-O: 20: paddi 3, 13, 0 ; CHECK-O-NEXT: 0000000000000020: R_PPC64_TPREL34 x ; CHECK-O-NEXT: 28: lwz 3, 0(3) ; CHECK-O-NEXT: 2c: blr @@ -43,11 +43,11 @@ entry: define dso_local void @LocalExecValueStore(i32 %in) { ; CHECK-S-LABEL: LocalExecValueStore: ; CHECK-S: # %bb.0: # %entry -; CHECK-S-NEXT: paddi r4, r13, x@TPREL, 0 +; CHECK-S-NEXT: paddi r4, r13, x@TPREL ; CHECK-S-NEXT: stw r3, 0(r4) ; CHECK-S-NEXT: blr ; CHECK-O-LABEL: : -; CHECK-O: 40: paddi 4, 13, 0, 0 +; CHECK-O: 40: paddi 4, 13, 0 ; CHECK-O-NEXT: 0000000000000040: R_PPC64_TPREL34 x ; CHECK-O-NEXT: 48: stw 3, 0(4) ; CHECK-O-NEXT: 4c: blr @@ -59,11 +59,11 @@ entry: define dso_local i32 @LocalExecValueLoadOffset() { ; CHECK-S-LABEL: LocalExecValueLoadOffset: ; CHECK-S: # %bb.0: # %entry -; CHECK-S-NEXT: paddi r3, r13, y@TPREL, 0 +; CHECK-S-NEXT: paddi r3, r13, y@TPREL ; CHECK-S-NEXT: lwz r3, 12(r3) ; CHECK-S-NEXT: blr ; CHECK-O-LABEL: : -; CHECK-O: 60: paddi 3, 13, 0, 0 +; CHECK-O: 60: paddi 3, 13, 0 ; CHECK-O-NEXT: 0000000000000060: R_PPC64_TPREL34 y ; CHECK-O-NEXT: 68: lwz 3, 12(3) ; CHECK-O-NEXT: 6c: blr @@ -76,11 +76,11 @@ entry: define dso_local ptr @LocalExecValueLoadOffsetNoLoad() { ; CHECK-S-LABEL: LocalExecValueLoadOffsetNoLoad: ; CHECK-S: # %bb.0: # %entry -; CHECK-S-NEXT: paddi r3, r13, y@TPREL, 0 +; CHECK-S-NEXT: paddi r3, r13, y@TPREL ; CHECK-S-NEXT: addi r3, r3, 12 ; CHECK-S-NEXT: blr ; CHECK-O-LABEL: : -; CHECK-O: 80: paddi 3, 13, 0, 0 +; CHECK-O: 80: paddi 3, 13, 0 ; CHECK-O-NEXT: 0000000000000080: R_PPC64_TPREL34 y ; CHECK-O-NEXT: 88: addi 3, 3, 12 ; CHECK-O-NEXT: 8c: blr diff --git a/llvm/test/CodeGen/PowerPC/pcrel-tls_get_addr_clobbers.ll b/llvm/test/CodeGen/PowerPC/pcrel-tls_get_addr_clobbers.ll index 38f55dac1ea8126d396c7e4c438f2ba108d5d5a6..905d13814ab7d8992890c5826180631de6365ce9 100644 --- a/llvm/test/CodeGen/PowerPC/pcrel-tls_get_addr_clobbers.ll +++ b/llvm/test/CodeGen/PowerPC/pcrel-tls_get_addr_clobbers.ll @@ -13,7 +13,7 @@ define void @test(ptr %arg) { ; CHECK-NEXT: mr r30, r3 ; CHECK-NEXT: paddi r3, 0, x@got@tlsld@pcrel, 1 ; CHECK-NEXT: bl __tls_get_addr@notoc(x@tlsld) -; CHECK-NEXT: paddi r3, r3, x@DTPREL, 0 +; CHECK-NEXT: paddi r3, r3, x@DTPREL ; CHECK-NEXT: std r30, 0(r3) ; CHECK-NEXT: addi r1, r1, 48 ; CHECK-NEXT: ld r0, 16(r1) diff --git a/llvm/test/CodeGen/PowerPC/tls-crash.mir b/llvm/test/CodeGen/PowerPC/tls-crash.mir index 186ba6da325a9189ab56300997299771c4d2d5cb..aa08790b36c9ec17f4357edd2555a85c82804e73 100644 --- a/llvm/test/CodeGen/PowerPC/tls-crash.mir +++ b/llvm/test/CodeGen/PowerPC/tls-crash.mir @@ -18,8 +18,8 @@ body: | liveins: $x3 %0:g8rc = COPY killed $x3 - %1:g8rc_and_g8rc_nox0 = PADDI8pc 0, target-flags(ppc-pcrel, ppc-got, ppc-tlsld) @x - %2:g8rc_and_g8rc_nox0 = PADDIdtprel killed %1, target-flags(ppc-pcrel, ppc-got, ppc-tlsld) @x + %1:g8rc_and_g8rc_nox0 = PADDI8pc 0, target-flags(ppc-got-tlsld-pcrel) @x + %2:g8rc_and_g8rc_nox0 = PADDIdtprel killed %1, target-flags(ppc-got-tlsld-pcrel) @x STD killed %0, 0, killed %2 :: (store (s64) into `i8** bitcast (%0** @x to i8**)`) BLR8 implicit $lr8, implicit $rm ... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vacopy.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vacopy.ll new file mode 100644 index 0000000000000000000000000000000000000000..48d72108335e469588c17100dfa044c364ca22ca --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vacopy.ll @@ -0,0 +1,28 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=riscv32 -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \ +; RUN: | FileCheck -check-prefix=RV32I %s +; RUN: llc -mtriple=riscv64 -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \ +; RUN: | FileCheck -check-prefix=RV64I %s + +declare void @llvm.va_copy(ptr, ptr) +define void @test_va_copy(ptr %dest_list, ptr %src_list) { + ; RV32I-LABEL: name: test_va_copy + ; RV32I: bb.1 (%ir-block.0): + ; RV32I-NEXT: liveins: $x10, $x11 + ; RV32I-NEXT: {{ $}} + ; RV32I-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32I-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x11 + ; RV32I-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[COPY]](p0), [[COPY1]](p0) + ; RV32I-NEXT: PseudoRET + ; + ; RV64I-LABEL: name: test_va_copy + ; RV64I: bb.1 (%ir-block.0): + ; RV64I-NEXT: liveins: $x10, $x11 + ; RV64I-NEXT: {{ $}} + ; RV64I-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV64I-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x11 + ; RV64I-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[COPY]](p0), [[COPY1]](p0) + ; RV64I-NEXT: PseudoRET + call void @llvm.va_copy(ptr %dest_list, ptr %src_list) + ret void +} diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vararg.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vararg.ll index e03ef07b5fc0dd369d16825f274640afe8c88ca5..ff30ebd3a8c74ecaffbc5242b86f48fc13cd8208 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vararg.ll +++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vararg.ll @@ -1,26 +1,27 @@ ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3 ; RUN: llc -mtriple=riscv32 -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \ -; RUN: | FileCheck -check-prefixes=ILP32 %s +; RUN: | FileCheck -check-prefixes=RV32,ILP32 %s ; RUN: llc -mtriple=riscv32 -global-isel -stop-after=irtranslator -mattr=+d -verify-machineinstrs < %s \ -; RUN: | FileCheck -check-prefixes=RV32D-ILP32 %s +; RUN: | FileCheck -check-prefixes=RV32,RV32D-ILP32 %s ; RUN: llc -mtriple=riscv32 -global-isel -stop-after=irtranslator -mattr=+d -target-abi ilp32f \ ; RUN: -verify-machineinstrs < %s \ -; RUN: | FileCheck -check-prefixes=RV32D-ILP32F %s +; RUN: | FileCheck -check-prefixes=RV32,RV32D-ILP32F %s ; RUN: llc -mtriple=riscv32 -global-isel -stop-after=irtranslator -mattr=+d -target-abi ilp32d \ ; RUN: -verify-machineinstrs < %s \ -; RUN: | FileCheck -check-prefixes=RV32D-ILP32D %s +; RUN: | FileCheck -check-prefixes=RV32,RV32D-ILP32D %s ; RUN: llc -mtriple=riscv64 -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \ -; RUN: | FileCheck -check-prefixes=LP64 %s +; RUN: | FileCheck -check-prefixes=RV64,LP64 %s ; RUN: llc -mtriple=riscv64 -global-isel -stop-after=irtranslator -mattr=+d -target-abi lp64f \ ; RUN: -verify-machineinstrs < %s \ -; RUN: | FileCheck -check-prefixes=LP64F %s +; RUN: | FileCheck -check-prefixes=RV64,LP64F %s ; RUN: llc -mtriple=riscv64 -global-isel -stop-after=irtranslator -mattr=+d -target-abi lp64d \ ; RUN: -verify-machineinstrs < %s \ -; RUN: | FileCheck -check-prefixes=LP64D %s +; RUN: | FileCheck -check-prefixes=RV64,LP64D %s ; The same vararg calling convention is used for ilp32/ilp32f/ilp32d and for -; lp64/lp64f/lp64d. Different CHECK lines are required for RV32D due to slight -; codegen differences due to the way the f64 load operations are lowered. +; lp64/lp64f/lp64d. Different CHECK lines are required due to slight +; codegen differences due to the way the f64 load operations are lowered and +; because the PseudoCALL specifies the calling convention. ; The nounwind attribute is omitted for some of the tests, to check that CFI ; directives are correctly generated. @@ -29,7 +30,505 @@ declare void @llvm.va_end(ptr) declare void @notdead(ptr) -declare i32 @va1(ptr %fmt, ...) +; Although frontends are recommended to not generate va_arg due to the lack of +; support for aggregate types, we test simple cases here to ensure they are +; lowered correctly + +define i32 @va1(ptr %fmt, ...) { + ; RV32-LABEL: name: va1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (dereferenceable load (p0) from %ir.va) + ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[LOAD]], [[C1]](s32) + ; RV32-NEXT: G_STORE [[PTR_ADD7]](p0), [[FRAME_INDEX1]](p0) :: (store (p0) into %ir.va) + ; RV32-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[LOAD]](p0) :: (load (s32) from %ir.argp.cur) + ; RV32-NEXT: $x10 = COPY [[LOAD1]](s32) + ; RV32-NEXT: PseudoRET implicit $x10 + ; + ; RV64-LABEL: name: va1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; RV64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (dereferenceable load (p0) from %ir.va, align 4) + ; RV64-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4 + ; RV64-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[LOAD]], [[C1]](s64) + ; RV64-NEXT: G_STORE [[PTR_ADD7]](p0), [[FRAME_INDEX1]](p0) :: (store (p0) into %ir.va, align 4) + ; RV64-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[LOAD]](p0) :: (load (s32) from %ir.argp.cur) + ; RV64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LOAD1]](s32) + ; RV64-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load ptr, ptr %va, align 4 + %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4 + store ptr %argp.next, ptr %va, align 4 + %1 = load i32, ptr %argp.cur, align 4 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + +; Ensure the adjustment when restoring the stack pointer using the frame +; pointer is correct +define i32 @va1_va_arg_alloca(ptr %fmt, ...) nounwind { + ; ILP32-LABEL: name: va1_va_arg_alloca + ; ILP32: bb.1 (%ir-block.0): + ; ILP32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; ILP32-NEXT: {{ $}} + ; ILP32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; ILP32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; ILP32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; ILP32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; ILP32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; ILP32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; ILP32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; ILP32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; ILP32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; ILP32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; ILP32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; ILP32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; ILP32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; ILP32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; ILP32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; ILP32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; ILP32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; ILP32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; ILP32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; ILP32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; ILP32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; ILP32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; ILP32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; ILP32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; ILP32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; ILP32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; ILP32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; ILP32-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; ILP32-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[VAARG]], [[C1]] + ; ILP32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 15 + ; ILP32-NEXT: [[ADD:%[0-9]+]]:_(s32) = nuw G_ADD [[MUL]], [[C2]] + ; ILP32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -16 + ; ILP32-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C3]] + ; ILP32-NEXT: [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s32), 1 + ; ILP32-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; ILP32-NEXT: $x10 = COPY [[DYN_STACKALLOC]](p0) + ; ILP32-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32_lp64, implicit-def $x1, implicit $x10 + ; ILP32-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; ILP32-NEXT: $x10 = COPY [[VAARG]](s32) + ; ILP32-NEXT: PseudoRET implicit $x10 + ; + ; RV32D-ILP32-LABEL: name: va1_va_arg_alloca + ; RV32D-ILP32: bb.1 (%ir-block.0): + ; RV32D-ILP32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32D-ILP32-NEXT: {{ $}} + ; RV32D-ILP32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32D-ILP32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32D-ILP32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32D-ILP32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32D-ILP32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32D-ILP32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32D-ILP32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32D-ILP32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32D-ILP32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32D-ILP32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32D-ILP32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32D-ILP32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32D-ILP32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32D-ILP32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32D-ILP32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32D-ILP32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32D-ILP32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32D-ILP32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; RV32D-ILP32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32D-ILP32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32D-ILP32-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[VAARG]], [[C1]] + ; RV32D-ILP32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 15 + ; RV32D-ILP32-NEXT: [[ADD:%[0-9]+]]:_(s32) = nuw G_ADD [[MUL]], [[C2]] + ; RV32D-ILP32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -16 + ; RV32D-ILP32-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C3]] + ; RV32D-ILP32-NEXT: [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s32), 1 + ; RV32D-ILP32-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32-NEXT: $x10 = COPY [[DYN_STACKALLOC]](p0) + ; RV32D-ILP32-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10 + ; RV32D-ILP32-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32-NEXT: $x10 = COPY [[VAARG]](s32) + ; RV32D-ILP32-NEXT: PseudoRET implicit $x10 + ; + ; RV32D-ILP32F-LABEL: name: va1_va_arg_alloca + ; RV32D-ILP32F: bb.1 (%ir-block.0): + ; RV32D-ILP32F-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32D-ILP32F-NEXT: {{ $}} + ; RV32D-ILP32F-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32D-ILP32F-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32D-ILP32F-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32D-ILP32F-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32D-ILP32F-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32D-ILP32F-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32D-ILP32F-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32D-ILP32F-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32D-ILP32F-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32D-ILP32F-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32D-ILP32F-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; RV32D-ILP32F-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32D-ILP32F-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32D-ILP32F-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32F-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[VAARG]], [[C1]] + ; RV32D-ILP32F-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 15 + ; RV32D-ILP32F-NEXT: [[ADD:%[0-9]+]]:_(s32) = nuw G_ADD [[MUL]], [[C2]] + ; RV32D-ILP32F-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -16 + ; RV32D-ILP32F-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C3]] + ; RV32D-ILP32F-NEXT: [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s32), 1 + ; RV32D-ILP32F-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32F-NEXT: $x10 = COPY [[DYN_STACKALLOC]](p0) + ; RV32D-ILP32F-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10 + ; RV32D-ILP32F-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32F-NEXT: $x10 = COPY [[VAARG]](s32) + ; RV32D-ILP32F-NEXT: PseudoRET implicit $x10 + ; + ; RV32D-ILP32D-LABEL: name: va1_va_arg_alloca + ; RV32D-ILP32D: bb.1 (%ir-block.0): + ; RV32D-ILP32D-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32D-ILP32D-NEXT: {{ $}} + ; RV32D-ILP32D-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32D-ILP32D-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32D-ILP32D-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32D-ILP32D-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32D-ILP32D-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32D-ILP32D-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32D-ILP32D-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32D-ILP32D-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32D-ILP32D-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32D-ILP32D-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32D-ILP32D-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; RV32D-ILP32D-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32D-ILP32D-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32D-ILP32D-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32D-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[VAARG]], [[C1]] + ; RV32D-ILP32D-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 15 + ; RV32D-ILP32D-NEXT: [[ADD:%[0-9]+]]:_(s32) = nuw G_ADD [[MUL]], [[C2]] + ; RV32D-ILP32D-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -16 + ; RV32D-ILP32D-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C3]] + ; RV32D-ILP32D-NEXT: [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s32), 1 + ; RV32D-ILP32D-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32D-NEXT: $x10 = COPY [[DYN_STACKALLOC]](p0) + ; RV32D-ILP32D-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10 + ; RV32D-ILP32D-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32D-NEXT: $x10 = COPY [[VAARG]](s32) + ; RV32D-ILP32D-NEXT: PseudoRET implicit $x10 + ; + ; LP64-LABEL: name: va1_va_arg_alloca + ; LP64: bb.1 (%ir-block.0): + ; LP64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; LP64-NEXT: {{ $}} + ; LP64-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; LP64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; LP64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; LP64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; LP64-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; LP64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; LP64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; LP64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; LP64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; LP64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; LP64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; LP64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; LP64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; LP64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; LP64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; LP64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; LP64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; LP64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; LP64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; LP64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; LP64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; LP64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; LP64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; LP64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; LP64-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; LP64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; LP64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; LP64-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[VAARG]](s32) + ; LP64-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ZEXT]], [[C1]] + ; LP64-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 15 + ; LP64-NEXT: [[ADD:%[0-9]+]]:_(s64) = nuw G_ADD [[MUL]], [[C2]] + ; LP64-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 -16 + ; LP64-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ADD]], [[C3]] + ; LP64-NEXT: [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s64), 1 + ; LP64-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; LP64-NEXT: $x10 = COPY [[DYN_STACKALLOC]](p0) + ; LP64-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32_lp64, implicit-def $x1, implicit $x10 + ; LP64-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; LP64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[VAARG]](s32) + ; LP64-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; LP64-NEXT: PseudoRET implicit $x10 + ; + ; LP64F-LABEL: name: va1_va_arg_alloca + ; LP64F: bb.1 (%ir-block.0): + ; LP64F-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; LP64F-NEXT: {{ $}} + ; LP64F-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; LP64F-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; LP64F-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; LP64F-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; LP64F-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; LP64F-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; LP64F-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; LP64F-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; LP64F-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; LP64F-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; LP64F-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; LP64F-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; LP64F-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; LP64F-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; LP64F-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; LP64F-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; LP64F-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; LP64F-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; LP64F-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; LP64F-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; LP64F-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; LP64F-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; LP64F-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; LP64F-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; LP64F-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; LP64F-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; LP64F-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; LP64F-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64F-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[VAARG]](s32) + ; LP64F-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ZEXT]], [[C1]] + ; LP64F-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 15 + ; LP64F-NEXT: [[ADD:%[0-9]+]]:_(s64) = nuw G_ADD [[MUL]], [[C2]] + ; LP64F-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 -16 + ; LP64F-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ADD]], [[C3]] + ; LP64F-NEXT: [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s64), 1 + ; LP64F-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; LP64F-NEXT: $x10 = COPY [[DYN_STACKALLOC]](p0) + ; LP64F-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10 + ; LP64F-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; LP64F-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[VAARG]](s32) + ; LP64F-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; LP64F-NEXT: PseudoRET implicit $x10 + ; + ; LP64D-LABEL: name: va1_va_arg_alloca + ; LP64D: bb.1 (%ir-block.0): + ; LP64D-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; LP64D-NEXT: {{ $}} + ; LP64D-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; LP64D-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; LP64D-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; LP64D-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; LP64D-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; LP64D-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; LP64D-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; LP64D-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; LP64D-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; LP64D-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; LP64D-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; LP64D-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; LP64D-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; LP64D-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; LP64D-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; LP64D-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; LP64D-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; LP64D-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; LP64D-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; LP64D-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; LP64D-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; LP64D-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; LP64D-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; LP64D-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; LP64D-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; LP64D-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; LP64D-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; LP64D-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64D-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[VAARG]](s32) + ; LP64D-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ZEXT]], [[C1]] + ; LP64D-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 15 + ; LP64D-NEXT: [[ADD:%[0-9]+]]:_(s64) = nuw G_ADD [[MUL]], [[C2]] + ; LP64D-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 -16 + ; LP64D-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ADD]], [[C3]] + ; LP64D-NEXT: [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s64), 1 + ; LP64D-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; LP64D-NEXT: $x10 = COPY [[DYN_STACKALLOC]](p0) + ; LP64D-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10 + ; LP64D-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; LP64D-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[VAARG]](s32) + ; LP64D-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; LP64D-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + %2 = alloca i8, i32 %1 + call void @notdead(ptr %2) + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + + +define i32 @va1_va_arg(ptr %fmt, ...) nounwind { + ; RV32-LABEL: name: va1_va_arg + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32-NEXT: $x10 = COPY [[VAARG]](s32) + ; RV32-NEXT: PseudoRET implicit $x10 + ; + ; RV64-LABEL: name: va1_va_arg + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; RV64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[VAARG]](s32) + ; RV64-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} define void @va1_caller() nounwind { ; ILP32-LABEL: name: va1_caller @@ -150,7 +649,188 @@ define void @va1_caller() nounwind { ; Ensure that 2x xlen size+alignment varargs are accessed via an "aligned" ; register pair (where the first register is even-numbered). -declare i64 @va2(ptr %fmt, ...) nounwind +define i64 @va2(ptr %fmt, ...) nounwind { + ; RV32-LABEL: name: va2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 7 + ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -8 + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (dereferenceable load (s32) from %ir.va) + ; RV32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD]], [[C1]] + ; RV32-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]] + ; RV32-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[ADD]](s32) + ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8 + ; RV32-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[INTTOPTR]], [[C3]](s32) + ; RV32-NEXT: G_STORE [[PTR_ADD7]](p0), [[FRAME_INDEX1]](p0) :: (store (p0) into %ir.va) + ; RV32-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s32) + ; RV32-NEXT: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[INTTOPTR1]](p0) :: (load (s64) from %ir.3) + ; RV32-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](s64) + ; RV32-NEXT: $x10 = COPY [[UV]](s32) + ; RV32-NEXT: $x11 = COPY [[UV1]](s32) + ; RV32-NEXT: PseudoRET implicit $x10, implicit $x11 + ; + ; RV64-LABEL: name: va2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; RV64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; RV64-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 7 + ; RV64-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -8 + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (dereferenceable load (s32) from %ir.va) + ; RV64-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD]], [[C1]] + ; RV64-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]] + ; RV64-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[ADD]](s32) + ; RV64-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[INTTOPTR]], [[C3]](s64) + ; RV64-NEXT: G_STORE [[PTR_ADD7]](p0), [[FRAME_INDEX1]](p0) :: (store (p0) into %ir.va, align 4) + ; RV64-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s32) + ; RV64-NEXT: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[INTTOPTR1]](p0) :: (load (s64) from %ir.3) + ; RV64-NEXT: $x10 = COPY [[LOAD1]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load i32, ptr %va, align 4 + %1 = add i32 %argp.cur, 7 + %2 = and i32 %1, -8 + %argp.cur.aligned = inttoptr i32 %1 to ptr + %argp.next = getelementptr inbounds i8, ptr %argp.cur.aligned, i32 8 + store ptr %argp.next, ptr %va, align 4 + %3 = inttoptr i32 %2 to ptr + %4 = load double, ptr %3, align 8 + %5 = bitcast double %4 to i64 + call void @llvm.va_end(ptr %va) + ret i64 %5 +} + +define i64 @va2_va_arg(ptr %fmt, ...) nounwind { + ; RV32-LABEL: name: va2_va_arg + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[VAARG:%[0-9]+]]:_(s64) = G_VAARG [[FRAME_INDEX1]](p0), 8 + ; RV32-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[VAARG]](s64) + ; RV32-NEXT: $x10 = COPY [[UV]](s32) + ; RV32-NEXT: $x11 = COPY [[UV1]](s32) + ; RV32-NEXT: PseudoRET implicit $x10, implicit $x11 + ; + ; RV64-LABEL: name: va2_va_arg + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; RV64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[VAARG:%[0-9]+]]:_(s64) = G_VAARG [[FRAME_INDEX1]](p0), 8 + ; RV64-NEXT: $x10 = COPY [[VAARG]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, double + call void @llvm.va_end(ptr %va) + %2 = bitcast double %1 to i64 + ret i64 %2 +} define void @va2_caller() nounwind { ; ILP32-LABEL: name: va2_caller @@ -259,7 +939,186 @@ define void @va2_caller() nounwind { ; On RV32, Ensure a named 2*xlen argument is passed in a1 and a2, while the ; vararg double is passed in a4 and a5 (rather than a3 and a4) -declare i64 @va3(i32 %a, i64 %b, ...) nounwind +define i64 @va3(i32 %a, i64 %b, ...) nounwind { + ; RV32-LABEL: name: va3 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32) + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 7 + ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -8 + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (dereferenceable load (s32) from %ir.va) + ; RV32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD]], [[C1]] + ; RV32-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]] + ; RV32-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[ADD]](s32) + ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8 + ; RV32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[INTTOPTR]], [[C3]](s32) + ; RV32-NEXT: G_STORE [[PTR_ADD5]](p0), [[FRAME_INDEX1]](p0) :: (store (p0) into %ir.va) + ; RV32-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s32) + ; RV32-NEXT: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[INTTOPTR1]](p0) :: (load (s64) from %ir.3) + ; RV32-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[MV]], [[LOAD1]] + ; RV32-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ADD1]](s64) + ; RV32-NEXT: $x10 = COPY [[UV]](s32) + ; RV32-NEXT: $x11 = COPY [[UV1]](s32) + ; RV32-NEXT: PseudoRET implicit $x10, implicit $x11 + ; + ; RV64-LABEL: name: va3 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; RV64-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64) + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.0, align 16) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.0 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.0 + 16, align 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.0 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.0 + 32, align 16) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.0 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 7 + ; RV64-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -8 + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (dereferenceable load (s32) from %ir.va) + ; RV64-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD]], [[C1]] + ; RV64-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]] + ; RV64-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[ADD]](s32) + ; RV64-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[INTTOPTR]], [[C3]](s64) + ; RV64-NEXT: G_STORE [[PTR_ADD6]](p0), [[FRAME_INDEX1]](p0) :: (store (p0) into %ir.va, align 4) + ; RV64-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s32) + ; RV64-NEXT: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[INTTOPTR1]](p0) :: (load (s64) from %ir.3) + ; RV64-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[COPY1]], [[LOAD1]] + ; RV64-NEXT: $x10 = COPY [[ADD1]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load i32, ptr %va, align 4 + %1 = add i32 %argp.cur, 7 + %2 = and i32 %1, -8 + %argp.cur.aligned = inttoptr i32 %1 to ptr + %argp.next = getelementptr inbounds i8, ptr %argp.cur.aligned, i32 8 + store ptr %argp.next, ptr %va, align 4 + %3 = inttoptr i32 %2 to ptr + %4 = load double, ptr %3, align 8 + call void @llvm.va_end(ptr %va) + %5 = bitcast double %4 to i64 + %6 = add i64 %b, %5 + ret i64 %6 +} + +define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind { + ; RV32-LABEL: name: va3_va_arg + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32) + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[VAARG:%[0-9]+]]:_(s64) = G_VAARG [[FRAME_INDEX1]](p0), 8 + ; RV32-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[MV]], [[VAARG]] + ; RV32-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ADD]](s64) + ; RV32-NEXT: $x10 = COPY [[UV]](s32) + ; RV32-NEXT: $x11 = COPY [[UV1]](s32) + ; RV32-NEXT: PseudoRET implicit $x10, implicit $x11 + ; + ; RV64-LABEL: name: va3_va_arg + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; RV64-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64) + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.0, align 16) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.0 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.0 + 16, align 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.0 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.0 + 32, align 16) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.0 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[VAARG:%[0-9]+]]:_(s64) = G_VAARG [[FRAME_INDEX1]](p0), 8 + ; RV64-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY1]], [[VAARG]] + ; RV64-NEXT: $x10 = COPY [[ADD]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, double + call void @llvm.va_end(ptr %va) + %2 = bitcast double %1 to i64 + %3 = add i64 %b, %2 + ret i64 %3 +} define void @va3_caller() nounwind { ; ILP32-LABEL: name: va3_caller @@ -391,3 +1250,532 @@ define void @va3_caller() nounwind { } declare void @llvm.va_copy(ptr, ptr) + +define i32 @va4_va_copy(i32 %argno, ...) nounwind { + ; ILP32-LABEL: name: va4_va_copy + ; ILP32: bb.1 (%ir-block.0): + ; ILP32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; ILP32-NEXT: {{ $}} + ; ILP32-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; ILP32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; ILP32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; ILP32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; ILP32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; ILP32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; ILP32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; ILP32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; ILP32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; ILP32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; ILP32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; ILP32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; ILP32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; ILP32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; ILP32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; ILP32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; ILP32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; ILP32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; ILP32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; ILP32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; ILP32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; ILP32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; ILP32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; ILP32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; ILP32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.vargs + ; ILP32-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.wargs + ; ILP32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.vargs) + ; ILP32-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; ILP32-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX2]](p0), [[FRAME_INDEX1]](p0) + ; ILP32-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.wargs) + ; ILP32-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; ILP32-NEXT: $x10 = COPY [[LOAD]](p0) + ; ILP32-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32_lp64, implicit-def $x1, implicit $x10 + ; ILP32-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; ILP32-NEXT: [[VAARG1:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; ILP32-NEXT: [[VAARG2:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; ILP32-NEXT: [[VAARG3:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; ILP32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[VAARG1]], [[VAARG]] + ; ILP32-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[VAARG2]] + ; ILP32-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[VAARG3]] + ; ILP32-NEXT: $x10 = COPY [[ADD2]](s32) + ; ILP32-NEXT: PseudoRET implicit $x10 + ; + ; RV32D-ILP32-LABEL: name: va4_va_copy + ; RV32D-ILP32: bb.1 (%ir-block.0): + ; RV32D-ILP32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32D-ILP32-NEXT: {{ $}} + ; RV32D-ILP32-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32D-ILP32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32D-ILP32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32D-ILP32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32D-ILP32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32D-ILP32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32D-ILP32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32D-ILP32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32D-ILP32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32D-ILP32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32D-ILP32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32D-ILP32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32D-ILP32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32D-ILP32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32D-ILP32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32D-ILP32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32D-ILP32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32D-ILP32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.vargs + ; RV32D-ILP32-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.wargs + ; RV32D-ILP32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.vargs) + ; RV32D-ILP32-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX2]](p0), [[FRAME_INDEX1]](p0) + ; RV32D-ILP32-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.wargs) + ; RV32D-ILP32-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32-NEXT: $x10 = COPY [[LOAD]](p0) + ; RV32D-ILP32-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10 + ; RV32D-ILP32-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32-NEXT: [[VAARG1:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32-NEXT: [[VAARG2:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32-NEXT: [[VAARG3:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[VAARG1]], [[VAARG]] + ; RV32D-ILP32-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[VAARG2]] + ; RV32D-ILP32-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[VAARG3]] + ; RV32D-ILP32-NEXT: $x10 = COPY [[ADD2]](s32) + ; RV32D-ILP32-NEXT: PseudoRET implicit $x10 + ; + ; RV32D-ILP32F-LABEL: name: va4_va_copy + ; RV32D-ILP32F: bb.1 (%ir-block.0): + ; RV32D-ILP32F-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32D-ILP32F-NEXT: {{ $}} + ; RV32D-ILP32F-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32D-ILP32F-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32D-ILP32F-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32D-ILP32F-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32D-ILP32F-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32D-ILP32F-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32D-ILP32F-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32D-ILP32F-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32D-ILP32F-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32D-ILP32F-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32D-ILP32F-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.vargs + ; RV32D-ILP32F-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.wargs + ; RV32D-ILP32F-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.vargs) + ; RV32D-ILP32F-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32F-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX2]](p0), [[FRAME_INDEX1]](p0) + ; RV32D-ILP32F-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.wargs) + ; RV32D-ILP32F-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32F-NEXT: $x10 = COPY [[LOAD]](p0) + ; RV32D-ILP32F-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10 + ; RV32D-ILP32F-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32F-NEXT: [[VAARG1:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32F-NEXT: [[VAARG2:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32F-NEXT: [[VAARG3:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32F-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[VAARG1]], [[VAARG]] + ; RV32D-ILP32F-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[VAARG2]] + ; RV32D-ILP32F-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[VAARG3]] + ; RV32D-ILP32F-NEXT: $x10 = COPY [[ADD2]](s32) + ; RV32D-ILP32F-NEXT: PseudoRET implicit $x10 + ; + ; RV32D-ILP32D-LABEL: name: va4_va_copy + ; RV32D-ILP32D: bb.1 (%ir-block.0): + ; RV32D-ILP32D-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32D-ILP32D-NEXT: {{ $}} + ; RV32D-ILP32D-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32D-ILP32D-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32D-ILP32D-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32D-ILP32D-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32D-ILP32D-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32D-ILP32D-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32D-ILP32D-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32D-ILP32D-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32D-ILP32D-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32D-ILP32D-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32D-ILP32D-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.vargs + ; RV32D-ILP32D-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.wargs + ; RV32D-ILP32D-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.vargs) + ; RV32D-ILP32D-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32D-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX2]](p0), [[FRAME_INDEX1]](p0) + ; RV32D-ILP32D-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.wargs) + ; RV32D-ILP32D-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32D-NEXT: $x10 = COPY [[LOAD]](p0) + ; RV32D-ILP32D-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10 + ; RV32D-ILP32D-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32D-NEXT: [[VAARG1:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32D-NEXT: [[VAARG2:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32D-NEXT: [[VAARG3:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32D-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[VAARG1]], [[VAARG]] + ; RV32D-ILP32D-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[VAARG2]] + ; RV32D-ILP32D-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[VAARG3]] + ; RV32D-ILP32D-NEXT: $x10 = COPY [[ADD2]](s32) + ; RV32D-ILP32D-NEXT: PseudoRET implicit $x10 + ; + ; LP64-LABEL: name: va4_va_copy + ; LP64: bb.1 (%ir-block.0): + ; LP64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; LP64-NEXT: {{ $}} + ; LP64-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; LP64-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64) + ; LP64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; LP64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; LP64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; LP64-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; LP64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; LP64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; LP64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; LP64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; LP64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; LP64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; LP64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; LP64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; LP64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; LP64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; LP64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; LP64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; LP64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; LP64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; LP64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; LP64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; LP64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; LP64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; LP64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; LP64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.vargs + ; LP64-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.wargs + ; LP64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.vargs) + ; LP64-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX2]](p0), [[FRAME_INDEX1]](p0) + ; LP64-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.wargs, align 4) + ; LP64-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; LP64-NEXT: $x10 = COPY [[LOAD]](p0) + ; LP64-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32_lp64, implicit-def $x1, implicit $x10 + ; LP64-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; LP64-NEXT: [[VAARG1:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64-NEXT: [[VAARG2:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64-NEXT: [[VAARG3:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[VAARG1]], [[VAARG]] + ; LP64-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[VAARG2]] + ; LP64-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[VAARG3]] + ; LP64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD2]](s32) + ; LP64-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; LP64-NEXT: PseudoRET implicit $x10 + ; + ; LP64F-LABEL: name: va4_va_copy + ; LP64F: bb.1 (%ir-block.0): + ; LP64F-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; LP64F-NEXT: {{ $}} + ; LP64F-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; LP64F-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64) + ; LP64F-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; LP64F-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; LP64F-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; LP64F-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; LP64F-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; LP64F-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; LP64F-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; LP64F-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; LP64F-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; LP64F-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; LP64F-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; LP64F-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; LP64F-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; LP64F-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; LP64F-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; LP64F-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; LP64F-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; LP64F-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; LP64F-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; LP64F-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; LP64F-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; LP64F-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; LP64F-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; LP64F-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.vargs + ; LP64F-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.wargs + ; LP64F-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.vargs) + ; LP64F-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64F-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX2]](p0), [[FRAME_INDEX1]](p0) + ; LP64F-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.wargs, align 4) + ; LP64F-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; LP64F-NEXT: $x10 = COPY [[LOAD]](p0) + ; LP64F-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10 + ; LP64F-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; LP64F-NEXT: [[VAARG1:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64F-NEXT: [[VAARG2:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64F-NEXT: [[VAARG3:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64F-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[VAARG1]], [[VAARG]] + ; LP64F-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[VAARG2]] + ; LP64F-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[VAARG3]] + ; LP64F-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD2]](s32) + ; LP64F-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; LP64F-NEXT: PseudoRET implicit $x10 + ; + ; LP64D-LABEL: name: va4_va_copy + ; LP64D: bb.1 (%ir-block.0): + ; LP64D-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; LP64D-NEXT: {{ $}} + ; LP64D-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; LP64D-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64) + ; LP64D-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; LP64D-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; LP64D-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; LP64D-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; LP64D-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; LP64D-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; LP64D-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; LP64D-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; LP64D-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; LP64D-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; LP64D-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; LP64D-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; LP64D-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; LP64D-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; LP64D-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; LP64D-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; LP64D-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; LP64D-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; LP64D-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; LP64D-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; LP64D-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; LP64D-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; LP64D-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; LP64D-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.vargs + ; LP64D-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.wargs + ; LP64D-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.vargs) + ; LP64D-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64D-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX2]](p0), [[FRAME_INDEX1]](p0) + ; LP64D-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.wargs, align 4) + ; LP64D-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; LP64D-NEXT: $x10 = COPY [[LOAD]](p0) + ; LP64D-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10 + ; LP64D-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; LP64D-NEXT: [[VAARG1:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64D-NEXT: [[VAARG2:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64D-NEXT: [[VAARG3:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64D-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[VAARG1]], [[VAARG]] + ; LP64D-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[VAARG2]] + ; LP64D-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[VAARG3]] + ; LP64D-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD2]](s32) + ; LP64D-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; LP64D-NEXT: PseudoRET implicit $x10 + %vargs = alloca ptr + %wargs = alloca ptr + call void @llvm.va_start(ptr %vargs) + %1 = va_arg ptr %vargs, i32 + call void @llvm.va_copy(ptr %wargs, ptr %vargs) + %2 = load ptr, ptr %wargs, align 4 + call void @notdead(ptr %2) + %3 = va_arg ptr %vargs, i32 + %4 = va_arg ptr %vargs, i32 + %5 = va_arg ptr %vargs, i32 + call void @llvm.va_end(ptr %vargs) + call void @llvm.va_end(ptr %wargs) + %add1 = add i32 %3, %1 + %add2 = add i32 %add1, %4 + %add3 = add i32 %add2, %5 + ret i32 %add3 +} + +; A function with no fixed arguments is not valid C, but can be +; specified in LLVM IR. We must ensure the vararg save area is +; still set up correctly. + +define i32 @va6_no_fixed_args(...) nounwind { + ; RV32-LABEL: name: va6_no_fixed_args + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32-NEXT: G_STORE [[COPY]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.0, align 16) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: G_STORE [[COPY1]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.0 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.0 + 8, align 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.0 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.0 + 16, align 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.0 + 20) + ; RV32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.0 + 24, align 8) + ; RV32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD6]](p0) :: (store (s32) into %fixed-stack.0 + 28) + ; RV32-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD6]], [[C]](s32) + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32-NEXT: $x10 = COPY [[VAARG]](s32) + ; RV32-NEXT: PseudoRET implicit $x10 + ; + ; RV64-LABEL: name: va6_no_fixed_args + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; RV64-NEXT: G_STORE [[COPY]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.0, align 16) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: G_STORE [[COPY1]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.0 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.0 + 16, align 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.0 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.0 + 32, align 16) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.0 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.0 + 48, align 16) + ; RV64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD6]](p0) :: (store (s64) into %fixed-stack.0 + 56) + ; RV64-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD6]], [[C]](s64) + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[VAARG]](s32) + ; RV64-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + +; TODO: improve constant materialization of stack addresses + +define i32 @va_large_stack(ptr %fmt, ...) { + ; RV32-LABEL: name: va_large_stack + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.large + ; RV32-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX2]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.va) + ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[LOAD]], [[C1]](s32) + ; RV32-NEXT: G_STORE [[PTR_ADD7]](p0), [[FRAME_INDEX2]](p0) :: (store (p0) into %ir.va) + ; RV32-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[LOAD]](p0) :: (load (s32) from %ir.argp.cur) + ; RV32-NEXT: $x10 = COPY [[LOAD1]](s32) + ; RV32-NEXT: PseudoRET implicit $x10 + ; + ; RV64-LABEL: name: va_large_stack + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; RV64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.large + ; RV64-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX2]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.va, align 4) + ; RV64-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4 + ; RV64-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[LOAD]], [[C1]](s64) + ; RV64-NEXT: G_STORE [[PTR_ADD7]](p0), [[FRAME_INDEX2]](p0) :: (store (p0) into %ir.va, align 4) + ; RV64-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[LOAD]](p0) :: (load (s32) from %ir.argp.cur) + ; RV64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LOAD1]](s32) + ; RV64-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %large = alloca [ 100000000 x i8 ] + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load ptr, ptr %va, align 4 + %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4 + store ptr %argp.next, ptr %va, align 4 + %1 = load i32, ptr %argp.cur, align 4 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vaarg-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vaarg-rv32.mir new file mode 100644 index 0000000000000000000000000000000000000000..b67691d0ef067d4406e45b237e1980310177ba39 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vaarg-rv32.mir @@ -0,0 +1,70 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - | FileCheck %s + +# On RISC-V, the MinStackArgumentAlignment is 1 and the ABI Alignment for p0 is +# greater than 1, so we will always generate code to adjust for this alignment. + +--- +name: va_arg_i32 +legalized: false +tracksRegLiveness: true +fixedStack: + - { id: 0, type: default, offset: 0, size: 4, alignment: 16, + isImmutable: true, isAliased: false } +stack: + - { id: 0, type: default, offset: 0, size: 4, alignment: 4 } +machineFunctionInfo: + varArgsFrameIndex: -1 + varArgsSaveSize: 0 +body: | + bb.1: + liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; CHECK-LABEL: name: va_arg_i32 + ; CHECK: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load (p0)) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3 + ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD]], [[C]](s32) + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -4 + ; CHECK-NEXT: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[PTR_ADD]], [[C1]](s32) + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTRMASK]], [[C2]](s32) + ; CHECK-NEXT: G_STORE [[PTR_ADD1]](p0), [[FRAME_INDEX]](p0) :: (store (p0)) + ; CHECK-NEXT: PseudoRET + %0:_(p0) = G_FRAME_INDEX %stack.0 + %1:_(s32) = G_VAARG %0(p0), 4 + PseudoRET +... +--- +name: va_arg_ptr +legalized: false +tracksRegLiveness: true +fixedStack: + - { id: 0, type: default, offset: 0, size: 4, alignment: 16, + isImmutable: true, isAliased: false } +stack: + - { id: 0, type: default, offset: 0, size: 4, alignment: 4 } +machineFunctionInfo: + varArgsFrameIndex: -1 + varArgsSaveSize: 0 +body: | + bb.1: + liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; CHECK-LABEL: name: va_arg_ptr + ; CHECK: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load (p0)) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3 + ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD]], [[C]](s32) + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -4 + ; CHECK-NEXT: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[PTR_ADD]], [[C1]](s32) + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTRMASK]], [[C2]](s32) + ; CHECK-NEXT: G_STORE [[PTR_ADD1]](p0), [[FRAME_INDEX]](p0) :: (store (p0)) + ; CHECK-NEXT: PseudoRET + %0:_(p0) = G_FRAME_INDEX %stack.0 + %1:_(p0) = G_VAARG %0(p0), 4 + PseudoRET +... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vaarg-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vaarg-rv64.mir new file mode 100644 index 0000000000000000000000000000000000000000..537c0fb14d260d859cbc4035378e7da555bd53d4 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vaarg-rv64.mir @@ -0,0 +1,93 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - | FileCheck %s + +# On RISC-V, the MinStackArgumentAlignment is 1 and the ABI Alignment for p0 is +# greater than 1, so we will always generate code to adjust for this alignment. + +--- +name: va_arg_i32 +legalized: false +tracksRegLiveness: true +fixedStack: + - { id: 0, type: default, offset: 0, size: 8, alignment: 16, + isImmutable: true, isAliased: false } +stack: + - { id: 0, type: default, offset: 0, size: 8, alignment: 8 } +machineFunctionInfo: + varArgsFrameIndex: -1 + varArgsSaveSize: 0 +body: | + bb.1: + ; CHECK-LABEL: name: va_arg_i32 + ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load (p0)) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3 + ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD]], [[C]](s64) + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -4 + ; CHECK-NEXT: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[PTR_ADD]], [[C1]](s64) + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4 + ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTRMASK]], [[C2]](s64) + ; CHECK-NEXT: G_STORE [[PTR_ADD1]](p0), [[FRAME_INDEX]](p0) :: (store (p0)) + ; CHECK-NEXT: PseudoRET + %0:_(p0) = G_FRAME_INDEX %stack.0 + %1:_(s32) = G_VAARG %0(p0), 4 + PseudoRET +... +--- +name: va_arg_i64 +legalized: false +tracksRegLiveness: true +fixedStack: + - { id: 0, type: default, offset: 0, size: 8, alignment: 16, + isImmutable: true, isAliased: false } +stack: + - { id: 0, type: default, offset: 0, size: 8, alignment: 8 } +machineFunctionInfo: + varArgsFrameIndex: -1 + varArgsSaveSize: 0 +body: | + bb.1: + ; CHECK-LABEL: name: va_arg_i64 + ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load (p0)) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3 + ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD]], [[C]](s64) + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -4 + ; CHECK-NEXT: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[PTR_ADD]], [[C1]](s64) + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTRMASK]], [[C2]](s64) + ; CHECK-NEXT: G_STORE [[PTR_ADD1]](p0), [[FRAME_INDEX]](p0) :: (store (p0)) + ; CHECK-NEXT: PseudoRET + %0:_(p0) = G_FRAME_INDEX %stack.0 + %1:_(s64) = G_VAARG %0(p0), 4 + PseudoRET +... +--- +name: va_arg_ptr +legalized: false +tracksRegLiveness: true +fixedStack: + - { id: 0, type: default, offset: 0, size: 8, alignment: 16, + isImmutable: true, isAliased: false } +stack: + - { id: 0, type: default, offset: 0, size: 8, alignment: 8 } +machineFunctionInfo: + varArgsFrameIndex: -1 + varArgsSaveSize: 0 +body: | + bb.1: + ; CHECK-LABEL: name: va_arg_ptr + ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load (p0)) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3 + ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD]], [[C]](s64) + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -4 + ; CHECK-NEXT: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[PTR_ADD]], [[C1]](s64) + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTRMASK]], [[C2]](s64) + ; CHECK-NEXT: G_STORE [[PTR_ADD1]](p0), [[FRAME_INDEX]](p0) :: (store (p0)) + ; CHECK-NEXT: PseudoRET + %0:_(p0) = G_FRAME_INDEX %stack.0 + %1:_(s64) = G_VAARG %0(p0), 4 + PseudoRET +... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vacopy.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vacopy.mir new file mode 100644 index 0000000000000000000000000000000000000000..f9eda1252937e87a428c128e0a00cae9d112c774 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vacopy.mir @@ -0,0 +1,23 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - | FileCheck %s +# RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - | FileCheck %s + +--- +name: test_va_copy +body: | + bb.1: + liveins: $x10, $x11 + + ; CHECK-LABEL: name: test_va_copy + ; CHECK: liveins: $x10, $x11 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x11 + ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[COPY1]](p0) :: (load (p0)) + ; CHECK-NEXT: G_STORE [[COPY]](p0), [[LOAD]](p0) :: (store (p0)) + ; CHECK-NEXT: PseudoRET + %0:_(p0) = COPY $x10 + %1:_(p0) = COPY $x11 + G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), %0(p0), %1(p0) + PseudoRET +... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vastart-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vastart-rv32.mir new file mode 100644 index 0000000000000000000000000000000000000000..181aa9cb9b315aba9aa35f94012387dafb962dd8 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vastart-rv32.mir @@ -0,0 +1,29 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - | FileCheck --check-prefix=RV32 %s + +--- +name: test_va_start +tracksRegLiveness: true +fixedStack: + - { id: 0, size: 4, alignment: 16, isImmutable: true } +stack: + - { id: 0, size: 4, alignment: 4 } +machineFunctionInfo: + varArgsFrameIndex: -1 + varArgsSaveSize: 0 +body: | + bb.1: + liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + + ; RV32-LABEL: name: test_va_start + ; RV32: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 + ; RV32-NEXT: G_STORE [[FRAME_INDEX1]](p0), [[FRAME_INDEX]](p0) :: (store (s32)) + ; RV32-NEXT: PseudoRET + %8:_(p0) = G_FRAME_INDEX %stack.0 + G_VASTART %8(p0) :: (store (s32)) + PseudoRET + +... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vastart-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vastart-rv64.mir new file mode 100644 index 0000000000000000000000000000000000000000..60babc87150a5d1dac560d22bbd4a55eac98d1f8 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vastart-rv64.mir @@ -0,0 +1,29 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - | FileCheck --check-prefix=RV64 %s + +--- +name: test_va_start +tracksRegLiveness: true +fixedStack: + - { id: 0, size: 8, alignment: 16, isImmutable: true } +stack: + - { id: 0, size: 8, alignment: 8 } +machineFunctionInfo: + varArgsFrameIndex: -1 + varArgsSaveSize: 0 +body: | + bb.1: + liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + + ; RV64-LABEL: name: test_va_start + ; RV64: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 + ; RV64-NEXT: G_STORE [[FRAME_INDEX1]](p0), [[FRAME_INDEX]](p0) :: (store (s64)) + ; RV64-NEXT: PseudoRET + %8:_(p0) = G_FRAME_INDEX %stack.0 + G_VASTART %8(p0) :: (store (s64)) + PseudoRET + +... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll b/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll new file mode 100644 index 0000000000000000000000000000000000000000..501a3c0ce74380d362c4a0f0204a59b93714922e --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll @@ -0,0 +1,1008 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -global-isel -verify-machineinstrs \ +; RUN: | FileCheck -check-prefixes=RV32,ILP32 %s +; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -global-isel -mattr=+d -verify-machineinstrs \ +; RUN: | FileCheck -check-prefixes=RV32,RV32D-ILP32 %s +; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -global-isel -mattr=+d -target-abi ilp32f \ +; RUN: -verify-machineinstrs \ +; RUN: | FileCheck -check-prefixes=RV32,RV32D-ILP32F %s +; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -global-isel -mattr=+d -target-abi ilp32d \ +; RUN: -verify-machineinstrs \ +; RUN: | FileCheck -check-prefixes=RV32,RV32D-ILP32D %s +; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -global-isel -verify-machineinstrs \ +; RUN: | FileCheck -check-prefixes=RV64,LP64 %s +; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -global-isel -mattr=+d -target-abi lp64f \ +; RUN: -verify-machineinstrs \ +; RUN: | FileCheck -check-prefixes=RV64,LP64F %s +; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -global-isel -mattr=+d -target-abi lp64d \ +; RUN: -verify-machineinstrs \ +; RUN: | FileCheck -check-prefixes=RV64,LP64D %s + +; The same vararg calling convention is used for ilp32/ilp32f/ilp32d and for +; lp64/lp64f/lp64d. Different CHECK lines are required due to slight +; codegen differences due to the way the f64 load operations are lowered and +; because the PseudoCALL specifies the calling convention. +; The nounwind attribute is omitted for some of the tests, to check that CFI +; directives are correctly generated. + +declare void @llvm.va_start(ptr) +declare void @llvm.va_end(ptr) + +declare void @notdead(ptr) + +; Although frontends are recommended to not generate va_arg due to the lack of +; support for aggregate types, we test simple cases here to ensure they are +; lowered correctly + +define i32 @va1(ptr %fmt, ...) { +; RV32-LABEL: va1: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -48 +; RV32-NEXT: .cfi_def_cfa_offset 48 +; RV32-NEXT: sw a1, 20(sp) +; RV32-NEXT: sw a2, 24(sp) +; RV32-NEXT: sw a3, 28(sp) +; RV32-NEXT: sw a4, 32(sp) +; RV32-NEXT: addi a0, sp, 20 +; RV32-NEXT: sw a0, 12(sp) +; RV32-NEXT: lw a0, 12(sp) +; RV32-NEXT: sw a5, 36(sp) +; RV32-NEXT: sw a6, 40(sp) +; RV32-NEXT: sw a7, 44(sp) +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, 12(sp) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: addi sp, sp, 48 +; RV32-NEXT: ret +; +; RV64-LABEL: va1: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -80 +; RV64-NEXT: .cfi_def_cfa_offset 80 +; RV64-NEXT: sd a1, 24(sp) +; RV64-NEXT: sd a2, 32(sp) +; RV64-NEXT: sd a3, 40(sp) +; RV64-NEXT: sd a4, 48(sp) +; RV64-NEXT: sd a5, 56(sp) +; RV64-NEXT: addi a0, sp, 24 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: lw a0, 12(sp) +; RV64-NEXT: lwu a1, 8(sp) +; RV64-NEXT: sd a6, 64(sp) +; RV64-NEXT: sd a7, 72(sp) +; RV64-NEXT: slli a0, a0, 32 +; RV64-NEXT: or a0, a0, a1 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: srli a2, a1, 32 +; RV64-NEXT: sw a1, 8(sp) +; RV64-NEXT: sw a2, 12(sp) +; RV64-NEXT: lw a0, 0(a0) +; RV64-NEXT: addi sp, sp, 80 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load ptr, ptr %va, align 4 + %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4 + store ptr %argp.next, ptr %va, align 4 + %1 = load i32, ptr %argp.cur, align 4 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + +define i32 @va1_va_arg(ptr %fmt, ...) nounwind { +; RV32-LABEL: va1_va_arg: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -48 +; RV32-NEXT: sw a1, 20(sp) +; RV32-NEXT: sw a2, 24(sp) +; RV32-NEXT: sw a3, 28(sp) +; RV32-NEXT: sw a4, 32(sp) +; RV32-NEXT: sw a5, 36(sp) +; RV32-NEXT: sw a6, 40(sp) +; RV32-NEXT: sw a7, 44(sp) +; RV32-NEXT: addi a0, sp, 20 +; RV32-NEXT: sw a0, 12(sp) +; RV32-NEXT: lw a0, 12(sp) +; RV32-NEXT: addi a0, a0, 3 +; RV32-NEXT: andi a0, a0, -4 +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, 12(sp) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: addi sp, sp, 48 +; RV32-NEXT: ret +; +; RV64-LABEL: va1_va_arg: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -80 +; RV64-NEXT: sd a1, 24(sp) +; RV64-NEXT: sd a2, 32(sp) +; RV64-NEXT: sd a3, 40(sp) +; RV64-NEXT: sd a4, 48(sp) +; RV64-NEXT: sd a5, 56(sp) +; RV64-NEXT: sd a6, 64(sp) +; RV64-NEXT: sd a7, 72(sp) +; RV64-NEXT: addi a0, sp, 24 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: addi a0, a0, 3 +; RV64-NEXT: andi a0, a0, -4 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: sd a1, 8(sp) +; RV64-NEXT: lw a0, 0(a0) +; RV64-NEXT: addi sp, sp, 80 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + +; Ensure the adjustment when restoring the stack pointer using the frame +; pointer is correct +define i32 @va1_va_arg_alloca(ptr %fmt, ...) nounwind { +; RV32-LABEL: va1_va_arg_alloca: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -48 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32-NEXT: addi s0, sp, 16 +; RV32-NEXT: sw a1, 4(s0) +; RV32-NEXT: sw a2, 8(s0) +; RV32-NEXT: sw a3, 12(s0) +; RV32-NEXT: sw a4, 16(s0) +; RV32-NEXT: sw a5, 20(s0) +; RV32-NEXT: sw a6, 24(s0) +; RV32-NEXT: sw a7, 28(s0) +; RV32-NEXT: addi a0, s0, 4 +; RV32-NEXT: sw a0, -16(s0) +; RV32-NEXT: lw a0, -16(s0) +; RV32-NEXT: addi a0, a0, 3 +; RV32-NEXT: andi a0, a0, -4 +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, -16(s0) +; RV32-NEXT: lw s1, 0(a0) +; RV32-NEXT: addi a0, s1, 15 +; RV32-NEXT: andi a0, a0, -16 +; RV32-NEXT: sub a0, sp, a0 +; RV32-NEXT: mv sp, a0 +; RV32-NEXT: call notdead@plt +; RV32-NEXT: mv a0, s1 +; RV32-NEXT: addi sp, s0, -16 +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 48 +; RV32-NEXT: ret +; +; RV64-LABEL: va1_va_arg_alloca: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -96 +; RV64-NEXT: sd ra, 24(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s0, 16(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s1, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: addi s0, sp, 32 +; RV64-NEXT: sd a1, 8(s0) +; RV64-NEXT: sd a2, 16(s0) +; RV64-NEXT: sd a3, 24(s0) +; RV64-NEXT: sd a4, 32(s0) +; RV64-NEXT: sd a5, 40(s0) +; RV64-NEXT: sd a6, 48(s0) +; RV64-NEXT: sd a7, 56(s0) +; RV64-NEXT: addi a0, s0, 8 +; RV64-NEXT: sd a0, -32(s0) +; RV64-NEXT: ld a0, -32(s0) +; RV64-NEXT: addi a0, a0, 3 +; RV64-NEXT: andi a0, a0, -4 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: sd a1, -32(s0) +; RV64-NEXT: lw s1, 0(a0) +; RV64-NEXT: slli a0, s1, 32 +; RV64-NEXT: srli a0, a0, 32 +; RV64-NEXT: addi a0, a0, 15 +; RV64-NEXT: andi a0, a0, -16 +; RV64-NEXT: sub a0, sp, a0 +; RV64-NEXT: mv sp, a0 +; RV64-NEXT: call notdead@plt +; RV64-NEXT: mv a0, s1 +; RV64-NEXT: addi sp, s0, -32 +; RV64-NEXT: ld ra, 24(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s0, 16(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s1, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 96 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + %2 = alloca i8, i32 %1 + call void @notdead(ptr %2) + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + +define void @va1_caller() nounwind { +; RV32-LABEL: va1_caller: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -16 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: lui a3, 261888 +; RV32-NEXT: li a4, 2 +; RV32-NEXT: li a2, 0 +; RV32-NEXT: call va1@plt +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 16 +; RV32-NEXT: ret +; +; LP64-LABEL: va1_caller: +; LP64: # %bb.0: +; LP64-NEXT: addi sp, sp, -16 +; LP64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; LP64-NEXT: lui a0, %hi(.LCPI3_0) +; LP64-NEXT: ld a1, %lo(.LCPI3_0)(a0) +; LP64-NEXT: li a2, 2 +; LP64-NEXT: call va1@plt +; LP64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; LP64-NEXT: addi sp, sp, 16 +; LP64-NEXT: ret +; +; LP64F-LABEL: va1_caller: +; LP64F: # %bb.0: +; LP64F-NEXT: addi sp, sp, -16 +; LP64F-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; LP64F-NEXT: li a0, 1023 +; LP64F-NEXT: slli a0, a0, 52 +; LP64F-NEXT: fmv.d.x fa5, a0 +; LP64F-NEXT: li a2, 2 +; LP64F-NEXT: fmv.x.d a1, fa5 +; LP64F-NEXT: call va1@plt +; LP64F-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; LP64F-NEXT: addi sp, sp, 16 +; LP64F-NEXT: ret +; +; LP64D-LABEL: va1_caller: +; LP64D: # %bb.0: +; LP64D-NEXT: addi sp, sp, -16 +; LP64D-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; LP64D-NEXT: li a0, 1023 +; LP64D-NEXT: slli a0, a0, 52 +; LP64D-NEXT: fmv.d.x fa5, a0 +; LP64D-NEXT: li a2, 2 +; LP64D-NEXT: fmv.x.d a1, fa5 +; LP64D-NEXT: call va1@plt +; LP64D-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; LP64D-NEXT: addi sp, sp, 16 +; LP64D-NEXT: ret + %1 = call i32 (ptr, ...) @va1(ptr undef, double 1.0, i32 2) + ret void +} + +; Ensure that 2x xlen size+alignment varargs are accessed via an "aligned" +; register pair (where the first register is even-numbered). + +define i64 @va2(ptr %fmt, ...) nounwind { +; ILP32-LABEL: va2: +; ILP32: # %bb.0: +; ILP32-NEXT: addi sp, sp, -48 +; ILP32-NEXT: sw a1, 20(sp) +; ILP32-NEXT: sw a2, 24(sp) +; ILP32-NEXT: sw a3, 28(sp) +; ILP32-NEXT: sw a4, 32(sp) +; ILP32-NEXT: addi a0, sp, 20 +; ILP32-NEXT: sw a0, 12(sp) +; ILP32-NEXT: lw a0, 12(sp) +; ILP32-NEXT: sw a5, 36(sp) +; ILP32-NEXT: sw a6, 40(sp) +; ILP32-NEXT: sw a7, 44(sp) +; ILP32-NEXT: addi a1, a0, 7 +; ILP32-NEXT: andi a1, a1, -8 +; ILP32-NEXT: addi a0, a0, 15 +; ILP32-NEXT: sw a0, 12(sp) +; ILP32-NEXT: lw a0, 0(a1) +; ILP32-NEXT: lw a1, 4(a1) +; ILP32-NEXT: addi sp, sp, 48 +; ILP32-NEXT: ret +; +; RV32D-ILP32-LABEL: va2: +; RV32D-ILP32: # %bb.0: +; RV32D-ILP32-NEXT: addi sp, sp, -48 +; RV32D-ILP32-NEXT: sw a1, 20(sp) +; RV32D-ILP32-NEXT: sw a2, 24(sp) +; RV32D-ILP32-NEXT: sw a3, 28(sp) +; RV32D-ILP32-NEXT: sw a4, 32(sp) +; RV32D-ILP32-NEXT: addi a0, sp, 20 +; RV32D-ILP32-NEXT: sw a0, 12(sp) +; RV32D-ILP32-NEXT: lw a0, 12(sp) +; RV32D-ILP32-NEXT: sw a5, 36(sp) +; RV32D-ILP32-NEXT: sw a6, 40(sp) +; RV32D-ILP32-NEXT: sw a7, 44(sp) +; RV32D-ILP32-NEXT: addi a1, a0, 7 +; RV32D-ILP32-NEXT: andi a1, a1, -8 +; RV32D-ILP32-NEXT: fld fa5, 0(a1) +; RV32D-ILP32-NEXT: addi a0, a0, 15 +; RV32D-ILP32-NEXT: sw a0, 12(sp) +; RV32D-ILP32-NEXT: fsd fa5, 0(sp) +; RV32D-ILP32-NEXT: lw a0, 0(sp) +; RV32D-ILP32-NEXT: lw a1, 4(sp) +; RV32D-ILP32-NEXT: addi sp, sp, 48 +; RV32D-ILP32-NEXT: ret +; +; RV32D-ILP32F-LABEL: va2: +; RV32D-ILP32F: # %bb.0: +; RV32D-ILP32F-NEXT: addi sp, sp, -48 +; RV32D-ILP32F-NEXT: sw a1, 20(sp) +; RV32D-ILP32F-NEXT: sw a2, 24(sp) +; RV32D-ILP32F-NEXT: sw a3, 28(sp) +; RV32D-ILP32F-NEXT: sw a4, 32(sp) +; RV32D-ILP32F-NEXT: addi a0, sp, 20 +; RV32D-ILP32F-NEXT: sw a0, 12(sp) +; RV32D-ILP32F-NEXT: lw a0, 12(sp) +; RV32D-ILP32F-NEXT: sw a5, 36(sp) +; RV32D-ILP32F-NEXT: sw a6, 40(sp) +; RV32D-ILP32F-NEXT: sw a7, 44(sp) +; RV32D-ILP32F-NEXT: addi a1, a0, 7 +; RV32D-ILP32F-NEXT: andi a1, a1, -8 +; RV32D-ILP32F-NEXT: fld fa5, 0(a1) +; RV32D-ILP32F-NEXT: addi a0, a0, 15 +; RV32D-ILP32F-NEXT: sw a0, 12(sp) +; RV32D-ILP32F-NEXT: fsd fa5, 0(sp) +; RV32D-ILP32F-NEXT: lw a0, 0(sp) +; RV32D-ILP32F-NEXT: lw a1, 4(sp) +; RV32D-ILP32F-NEXT: addi sp, sp, 48 +; RV32D-ILP32F-NEXT: ret +; +; RV32D-ILP32D-LABEL: va2: +; RV32D-ILP32D: # %bb.0: +; RV32D-ILP32D-NEXT: addi sp, sp, -48 +; RV32D-ILP32D-NEXT: sw a1, 20(sp) +; RV32D-ILP32D-NEXT: sw a2, 24(sp) +; RV32D-ILP32D-NEXT: sw a3, 28(sp) +; RV32D-ILP32D-NEXT: sw a4, 32(sp) +; RV32D-ILP32D-NEXT: addi a0, sp, 20 +; RV32D-ILP32D-NEXT: sw a0, 12(sp) +; RV32D-ILP32D-NEXT: lw a0, 12(sp) +; RV32D-ILP32D-NEXT: sw a5, 36(sp) +; RV32D-ILP32D-NEXT: sw a6, 40(sp) +; RV32D-ILP32D-NEXT: sw a7, 44(sp) +; RV32D-ILP32D-NEXT: addi a1, a0, 7 +; RV32D-ILP32D-NEXT: andi a1, a1, -8 +; RV32D-ILP32D-NEXT: fld fa5, 0(a1) +; RV32D-ILP32D-NEXT: addi a0, a0, 15 +; RV32D-ILP32D-NEXT: sw a0, 12(sp) +; RV32D-ILP32D-NEXT: fsd fa5, 0(sp) +; RV32D-ILP32D-NEXT: lw a0, 0(sp) +; RV32D-ILP32D-NEXT: lw a1, 4(sp) +; RV32D-ILP32D-NEXT: addi sp, sp, 48 +; RV32D-ILP32D-NEXT: ret +; +; RV64-LABEL: va2: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -80 +; RV64-NEXT: sd a1, 24(sp) +; RV64-NEXT: sd a2, 32(sp) +; RV64-NEXT: sd a3, 40(sp) +; RV64-NEXT: sd a4, 48(sp) +; RV64-NEXT: addi a0, sp, 24 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: sd a5, 56(sp) +; RV64-NEXT: sd a6, 64(sp) +; RV64-NEXT: sd a7, 72(sp) +; RV64-NEXT: addi a1, a0, 7 +; RV64-NEXT: andi a1, a1, -8 +; RV64-NEXT: addi a0, a0, 15 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 0(a1) +; RV64-NEXT: addi sp, sp, 80 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load ptr, ptr %va + %ptrint = ptrtoint ptr %argp.cur to iXLen + %1 = add iXLen %ptrint, 7 + %2 = and iXLen %1, -8 + %argp.cur.aligned = inttoptr iXLen %1 to ptr + %argp.next = getelementptr inbounds i8, ptr %argp.cur.aligned, i32 8 + store ptr %argp.next, ptr %va + %3 = inttoptr iXLen %2 to ptr + %4 = load double, ptr %3, align 8 + %5 = bitcast double %4 to i64 + call void @llvm.va_end(ptr %va) + ret i64 %5 +} + +; This test is slightly different than the SelectionDAG counterpart because +; narrowScalar and widenScalar for G_VAARG on types outside of [s32, sXLen] +; are not implemented yet. +define i64 @va2_va_arg(ptr %fmt, ...) nounwind { +; RV32-LABEL: va2_va_arg: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -48 +; RV32-NEXT: sw a1, 20(sp) +; RV32-NEXT: sw a2, 24(sp) +; RV32-NEXT: sw a3, 28(sp) +; RV32-NEXT: sw a4, 32(sp) +; RV32-NEXT: sw a5, 36(sp) +; RV32-NEXT: sw a6, 40(sp) +; RV32-NEXT: sw a7, 44(sp) +; RV32-NEXT: addi a0, sp, 20 +; RV32-NEXT: sw a0, 12(sp) +; RV32-NEXT: lw a0, 12(sp) +; RV32-NEXT: addi a0, a0, 3 +; RV32-NEXT: andi a0, a0, -4 +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, 12(sp) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: li a1, 0 +; RV32-NEXT: addi sp, sp, 48 +; RV32-NEXT: ret +; +; RV64-LABEL: va2_va_arg: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -80 +; RV64-NEXT: sd a1, 24(sp) +; RV64-NEXT: sd a2, 32(sp) +; RV64-NEXT: sd a3, 40(sp) +; RV64-NEXT: sd a4, 48(sp) +; RV64-NEXT: sd a5, 56(sp) +; RV64-NEXT: sd a6, 64(sp) +; RV64-NEXT: sd a7, 72(sp) +; RV64-NEXT: addi a0, sp, 24 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: addi a0, a0, 3 +; RV64-NEXT: andi a0, a0, -4 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: sd a1, 8(sp) +; RV64-NEXT: lw a0, 0(a0) +; RV64-NEXT: slli a0, a0, 32 +; RV64-NEXT: srli a0, a0, 32 +; RV64-NEXT: addi sp, sp, 80 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + call void @llvm.va_end(ptr %va) + %2 = zext i32 %1 to i64 + ret i64 %2 +} + +define void @va2_caller() nounwind { +; RV32-LABEL: va2_caller: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -16 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: li a1, 1 +; RV32-NEXT: call va2@plt +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 16 +; RV32-NEXT: ret +; +; RV64-LABEL: va2_caller: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -16 +; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: li a1, 1 +; RV64-NEXT: call va2@plt +; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 16 +; RV64-NEXT: ret + %1 = call i64 (ptr, ...) @va2(ptr undef, i32 1) + ret void +} + +; On RV32, Ensure a named 2*xlen argument is passed in a1 and a2, while the +; vararg double is passed in a4 and a5 (rather than a3 and a4) + +define i64 @va3(i32 %a, i64 %b, ...) nounwind { +; ILP32-LABEL: va3: +; ILP32: # %bb.0: +; ILP32-NEXT: addi sp, sp, -32 +; ILP32-NEXT: sw a3, 12(sp) +; ILP32-NEXT: sw a4, 16(sp) +; ILP32-NEXT: addi a0, sp, 12 +; ILP32-NEXT: sw a0, 4(sp) +; ILP32-NEXT: lw a0, 4(sp) +; ILP32-NEXT: sw a5, 20(sp) +; ILP32-NEXT: sw a6, 24(sp) +; ILP32-NEXT: sw a7, 28(sp) +; ILP32-NEXT: addi a3, a0, 7 +; ILP32-NEXT: andi a3, a3, -8 +; ILP32-NEXT: addi a0, a0, 15 +; ILP32-NEXT: sw a0, 4(sp) +; ILP32-NEXT: lw a4, 0(a3) +; ILP32-NEXT: lw a3, 4(a3) +; ILP32-NEXT: add a0, a1, a4 +; ILP32-NEXT: sltu a1, a0, a4 +; ILP32-NEXT: add a2, a2, a3 +; ILP32-NEXT: add a1, a2, a1 +; ILP32-NEXT: addi sp, sp, 32 +; ILP32-NEXT: ret +; +; RV32D-ILP32-LABEL: va3: +; RV32D-ILP32: # %bb.0: +; RV32D-ILP32-NEXT: addi sp, sp, -48 +; RV32D-ILP32-NEXT: sw a3, 28(sp) +; RV32D-ILP32-NEXT: sw a4, 32(sp) +; RV32D-ILP32-NEXT: addi a0, sp, 28 +; RV32D-ILP32-NEXT: sw a0, 20(sp) +; RV32D-ILP32-NEXT: lw a0, 20(sp) +; RV32D-ILP32-NEXT: sw a5, 36(sp) +; RV32D-ILP32-NEXT: sw a6, 40(sp) +; RV32D-ILP32-NEXT: sw a7, 44(sp) +; RV32D-ILP32-NEXT: addi a3, a0, 7 +; RV32D-ILP32-NEXT: andi a3, a3, -8 +; RV32D-ILP32-NEXT: fld fa5, 0(a3) +; RV32D-ILP32-NEXT: addi a0, a0, 15 +; RV32D-ILP32-NEXT: sw a0, 20(sp) +; RV32D-ILP32-NEXT: fsd fa5, 8(sp) +; RV32D-ILP32-NEXT: lw a3, 8(sp) +; RV32D-ILP32-NEXT: lw a4, 12(sp) +; RV32D-ILP32-NEXT: add a0, a1, a3 +; RV32D-ILP32-NEXT: sltu a1, a0, a3 +; RV32D-ILP32-NEXT: add a2, a2, a4 +; RV32D-ILP32-NEXT: add a1, a2, a1 +; RV32D-ILP32-NEXT: addi sp, sp, 48 +; RV32D-ILP32-NEXT: ret +; +; RV32D-ILP32F-LABEL: va3: +; RV32D-ILP32F: # %bb.0: +; RV32D-ILP32F-NEXT: addi sp, sp, -48 +; RV32D-ILP32F-NEXT: sw a3, 28(sp) +; RV32D-ILP32F-NEXT: sw a4, 32(sp) +; RV32D-ILP32F-NEXT: addi a0, sp, 28 +; RV32D-ILP32F-NEXT: sw a0, 20(sp) +; RV32D-ILP32F-NEXT: lw a0, 20(sp) +; RV32D-ILP32F-NEXT: sw a5, 36(sp) +; RV32D-ILP32F-NEXT: sw a6, 40(sp) +; RV32D-ILP32F-NEXT: sw a7, 44(sp) +; RV32D-ILP32F-NEXT: addi a3, a0, 7 +; RV32D-ILP32F-NEXT: andi a3, a3, -8 +; RV32D-ILP32F-NEXT: fld fa5, 0(a3) +; RV32D-ILP32F-NEXT: addi a0, a0, 15 +; RV32D-ILP32F-NEXT: sw a0, 20(sp) +; RV32D-ILP32F-NEXT: fsd fa5, 8(sp) +; RV32D-ILP32F-NEXT: lw a3, 8(sp) +; RV32D-ILP32F-NEXT: lw a4, 12(sp) +; RV32D-ILP32F-NEXT: add a0, a1, a3 +; RV32D-ILP32F-NEXT: sltu a1, a0, a3 +; RV32D-ILP32F-NEXT: add a2, a2, a4 +; RV32D-ILP32F-NEXT: add a1, a2, a1 +; RV32D-ILP32F-NEXT: addi sp, sp, 48 +; RV32D-ILP32F-NEXT: ret +; +; RV32D-ILP32D-LABEL: va3: +; RV32D-ILP32D: # %bb.0: +; RV32D-ILP32D-NEXT: addi sp, sp, -48 +; RV32D-ILP32D-NEXT: sw a3, 28(sp) +; RV32D-ILP32D-NEXT: sw a4, 32(sp) +; RV32D-ILP32D-NEXT: addi a0, sp, 28 +; RV32D-ILP32D-NEXT: sw a0, 20(sp) +; RV32D-ILP32D-NEXT: lw a0, 20(sp) +; RV32D-ILP32D-NEXT: sw a5, 36(sp) +; RV32D-ILP32D-NEXT: sw a6, 40(sp) +; RV32D-ILP32D-NEXT: sw a7, 44(sp) +; RV32D-ILP32D-NEXT: addi a3, a0, 7 +; RV32D-ILP32D-NEXT: andi a3, a3, -8 +; RV32D-ILP32D-NEXT: fld fa5, 0(a3) +; RV32D-ILP32D-NEXT: addi a0, a0, 15 +; RV32D-ILP32D-NEXT: sw a0, 20(sp) +; RV32D-ILP32D-NEXT: fsd fa5, 8(sp) +; RV32D-ILP32D-NEXT: lw a3, 8(sp) +; RV32D-ILP32D-NEXT: lw a4, 12(sp) +; RV32D-ILP32D-NEXT: add a0, a1, a3 +; RV32D-ILP32D-NEXT: sltu a1, a0, a3 +; RV32D-ILP32D-NEXT: add a2, a2, a4 +; RV32D-ILP32D-NEXT: add a1, a2, a1 +; RV32D-ILP32D-NEXT: addi sp, sp, 48 +; RV32D-ILP32D-NEXT: ret +; +; RV64-LABEL: va3: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -64 +; RV64-NEXT: sd a2, 16(sp) +; RV64-NEXT: sd a3, 24(sp) +; RV64-NEXT: sd a4, 32(sp) +; RV64-NEXT: addi a0, sp, 16 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: sd a5, 40(sp) +; RV64-NEXT: sd a6, 48(sp) +; RV64-NEXT: sd a7, 56(sp) +; RV64-NEXT: addi a2, a0, 7 +; RV64-NEXT: andi a2, a2, -8 +; RV64-NEXT: addi a0, a0, 15 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 0(a2) +; RV64-NEXT: add a0, a1, a0 +; RV64-NEXT: addi sp, sp, 64 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load ptr, ptr %va + %ptrint = ptrtoint ptr %argp.cur to iXLen + %1 = add iXLen %ptrint, 7 + %2 = and iXLen %1, -8 + %argp.cur.aligned = inttoptr iXLen %1 to ptr + %argp.next = getelementptr inbounds i8, ptr %argp.cur.aligned, i32 8 + store ptr %argp.next, ptr %va + %3 = inttoptr iXLen %2 to ptr + %4 = load double, ptr %3, align 8 + call void @llvm.va_end(ptr %va) + %5 = bitcast double %4 to i64 + %6 = add i64 %b, %5 + ret i64 %6 +} + +; This test is slightly different than the SelectionDAG counterpart because +; narrowScalar and widenScalar for G_VAARG on types outside of [s32, sXLen] +; are not implemented yet. +define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind { +; RV32-LABEL: va3_va_arg: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -32 +; RV32-NEXT: sw a3, 12(sp) +; RV32-NEXT: sw a4, 16(sp) +; RV32-NEXT: sw a5, 20(sp) +; RV32-NEXT: sw a6, 24(sp) +; RV32-NEXT: sw a7, 28(sp) +; RV32-NEXT: addi a0, sp, 12 +; RV32-NEXT: sw a0, 4(sp) +; RV32-NEXT: lw a0, 4(sp) +; RV32-NEXT: addi a0, a0, 3 +; RV32-NEXT: andi a0, a0, -4 +; RV32-NEXT: addi a3, a0, 4 +; RV32-NEXT: sw a3, 4(sp) +; RV32-NEXT: lw a3, 0(a0) +; RV32-NEXT: add a0, a1, a3 +; RV32-NEXT: sltu a1, a0, a3 +; RV32-NEXT: add a1, a2, a1 +; RV32-NEXT: addi sp, sp, 32 +; RV32-NEXT: ret +; +; RV64-LABEL: va3_va_arg: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -64 +; RV64-NEXT: sd a2, 16(sp) +; RV64-NEXT: sd a3, 24(sp) +; RV64-NEXT: sd a4, 32(sp) +; RV64-NEXT: sd a5, 40(sp) +; RV64-NEXT: sd a6, 48(sp) +; RV64-NEXT: sd a7, 56(sp) +; RV64-NEXT: addi a0, sp, 16 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: addi a0, a0, 3 +; RV64-NEXT: andi a0, a0, -4 +; RV64-NEXT: addi a2, a0, 4 +; RV64-NEXT: sd a2, 8(sp) +; RV64-NEXT: lw a0, 0(a0) +; RV64-NEXT: slli a0, a0, 32 +; RV64-NEXT: srli a0, a0, 32 +; RV64-NEXT: add a0, a1, a0 +; RV64-NEXT: addi sp, sp, 64 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + call void @llvm.va_end(ptr %va) + %2 = zext i32 %1 to i64 + %3 = add i64 %b, %2 + ret i64 %3 +} + +define void @va3_caller() nounwind { +; RV32-LABEL: va3_caller: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -16 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: lui a0, 5 +; RV32-NEXT: addi a3, a0, -480 +; RV32-NEXT: li a0, 2 +; RV32-NEXT: li a1, 1111 +; RV32-NEXT: li a2, 0 +; RV32-NEXT: call va3@plt +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 16 +; RV32-NEXT: ret +; +; RV64-LABEL: va3_caller: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -16 +; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: lui a0, 5 +; RV64-NEXT: addiw a2, a0, -480 +; RV64-NEXT: li a0, 2 +; RV64-NEXT: li a1, 1111 +; RV64-NEXT: call va3@plt +; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 16 +; RV64-NEXT: ret + %1 = call i64 (i32, i64, ...) @va3(i32 2, i64 1111, i32 20000) + ret void +} + +declare void @llvm.va_copy(ptr, ptr) + +define i32 @va4_va_copy(i32 %argno, ...) nounwind { +; RV32-LABEL: va4_va_copy: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -48 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32-NEXT: sw a1, 20(sp) +; RV32-NEXT: sw a2, 24(sp) +; RV32-NEXT: sw a3, 28(sp) +; RV32-NEXT: sw a4, 32(sp) +; RV32-NEXT: sw a5, 36(sp) +; RV32-NEXT: sw a6, 40(sp) +; RV32-NEXT: sw a7, 44(sp) +; RV32-NEXT: addi a0, sp, 20 +; RV32-NEXT: sw a0, 4(sp) +; RV32-NEXT: lw a0, 4(sp) +; RV32-NEXT: addi a0, a0, 3 +; RV32-NEXT: andi a0, a0, -4 +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, 4(sp) +; RV32-NEXT: lw a1, 4(sp) +; RV32-NEXT: mv a2, sp +; RV32-NEXT: lw s0, 0(a0) +; RV32-NEXT: sw a2, 0(a1) +; RV32-NEXT: lw a0, 0(sp) +; RV32-NEXT: call notdead@plt +; RV32-NEXT: lw a0, 4(sp) +; RV32-NEXT: addi a0, a0, 3 +; RV32-NEXT: andi a0, a0, -4 +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, 4(sp) +; RV32-NEXT: lw a1, 4(sp) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: addi a1, a1, 3 +; RV32-NEXT: andi a1, a1, -4 +; RV32-NEXT: addi a2, a1, 4 +; RV32-NEXT: sw a2, 4(sp) +; RV32-NEXT: lw a2, 4(sp) +; RV32-NEXT: lw a1, 0(a1) +; RV32-NEXT: addi a2, a2, 3 +; RV32-NEXT: andi a2, a2, -4 +; RV32-NEXT: addi a3, a2, 4 +; RV32-NEXT: sw a3, 4(sp) +; RV32-NEXT: lw a2, 0(a2) +; RV32-NEXT: add a0, a0, s0 +; RV32-NEXT: add a1, a1, a2 +; RV32-NEXT: add a0, a0, a1 +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 48 +; RV32-NEXT: ret +; +; RV64-LABEL: va4_va_copy: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -96 +; RV64-NEXT: sd ra, 24(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s0, 16(sp) # 8-byte Folded Spill +; RV64-NEXT: sd a1, 40(sp) +; RV64-NEXT: sd a2, 48(sp) +; RV64-NEXT: sd a3, 56(sp) +; RV64-NEXT: sd a4, 64(sp) +; RV64-NEXT: sd a5, 72(sp) +; RV64-NEXT: sd a6, 80(sp) +; RV64-NEXT: sd a7, 88(sp) +; RV64-NEXT: addi a0, sp, 40 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: addi a0, a0, 3 +; RV64-NEXT: andi a0, a0, -4 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: sd a1, 8(sp) +; RV64-NEXT: ld a1, 8(sp) +; RV64-NEXT: mv a2, sp +; RV64-NEXT: lw s0, 0(a0) +; RV64-NEXT: sd a2, 0(a1) +; RV64-NEXT: lw a0, 4(sp) +; RV64-NEXT: lwu a1, 0(sp) +; RV64-NEXT: slli a0, a0, 32 +; RV64-NEXT: or a0, a0, a1 +; RV64-NEXT: call notdead@plt +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: addi a0, a0, 3 +; RV64-NEXT: andi a0, a0, -4 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: sd a1, 8(sp) +; RV64-NEXT: ld a1, 8(sp) +; RV64-NEXT: lw a0, 0(a0) +; RV64-NEXT: addi a1, a1, 3 +; RV64-NEXT: andi a1, a1, -4 +; RV64-NEXT: addi a2, a1, 4 +; RV64-NEXT: sd a2, 8(sp) +; RV64-NEXT: ld a2, 8(sp) +; RV64-NEXT: lw a1, 0(a1) +; RV64-NEXT: addi a2, a2, 3 +; RV64-NEXT: andi a2, a2, -4 +; RV64-NEXT: addi a3, a2, 4 +; RV64-NEXT: sd a3, 8(sp) +; RV64-NEXT: lw a2, 0(a2) +; RV64-NEXT: add a0, a0, s0 +; RV64-NEXT: add a1, a1, a2 +; RV64-NEXT: addw a0, a0, a1 +; RV64-NEXT: ld ra, 24(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s0, 16(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 96 +; RV64-NEXT: ret + %vargs = alloca ptr + %wargs = alloca ptr + call void @llvm.va_start(ptr %vargs) + %1 = va_arg ptr %vargs, i32 + call void @llvm.va_copy(ptr %wargs, ptr %vargs) + %2 = load ptr, ptr %wargs, align 4 + call void @notdead(ptr %2) + %3 = va_arg ptr %vargs, i32 + %4 = va_arg ptr %vargs, i32 + %5 = va_arg ptr %vargs, i32 + call void @llvm.va_end(ptr %vargs) + call void @llvm.va_end(ptr %wargs) + %add1 = add i32 %3, %1 + %add2 = add i32 %add1, %4 + %add3 = add i32 %add2, %5 + ret i32 %add3 +} + +; The va5_aligned_stack_callee and caller function are ommitted from this file +; since they were not included in the IR lowering test when vararg calls were +; initially added. + +; A function with no fixed arguments is not valid C, but can be +; specified in LLVM IR. We must ensure the vararg save area is +; still set up correctly. + +define i32 @va6_no_fixed_args(...) nounwind { +; RV32-LABEL: va6_no_fixed_args: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -48 +; RV32-NEXT: sw a0, 16(sp) +; RV32-NEXT: sw a1, 20(sp) +; RV32-NEXT: sw a2, 24(sp) +; RV32-NEXT: sw a3, 28(sp) +; RV32-NEXT: sw a4, 32(sp) +; RV32-NEXT: sw a5, 36(sp) +; RV32-NEXT: sw a6, 40(sp) +; RV32-NEXT: sw a7, 44(sp) +; RV32-NEXT: addi a0, sp, 16 +; RV32-NEXT: sw a0, 12(sp) +; RV32-NEXT: lw a0, 12(sp) +; RV32-NEXT: addi a0, a0, 3 +; RV32-NEXT: andi a0, a0, -4 +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, 12(sp) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: addi sp, sp, 48 +; RV32-NEXT: ret +; +; RV64-LABEL: va6_no_fixed_args: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -80 +; RV64-NEXT: sd a0, 16(sp) +; RV64-NEXT: sd a1, 24(sp) +; RV64-NEXT: sd a2, 32(sp) +; RV64-NEXT: sd a3, 40(sp) +; RV64-NEXT: sd a4, 48(sp) +; RV64-NEXT: sd a5, 56(sp) +; RV64-NEXT: sd a6, 64(sp) +; RV64-NEXT: sd a7, 72(sp) +; RV64-NEXT: addi a0, sp, 16 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: addi a0, a0, 3 +; RV64-NEXT: andi a0, a0, -4 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: sd a1, 8(sp) +; RV64-NEXT: lw a0, 0(a0) +; RV64-NEXT: addi sp, sp, 80 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + +; TODO: improve constant materialization of stack addresses + +define i32 @va_large_stack(ptr %fmt, ...) { +; RV32-LABEL: va_large_stack: +; RV32: # %bb.0: +; RV32-NEXT: lui a0, 24414 +; RV32-NEXT: addi a0, a0, 304 +; RV32-NEXT: sub sp, sp, a0 +; RV32-NEXT: .cfi_def_cfa_offset 100000048 +; RV32-NEXT: lui a0, 24414 +; RV32-NEXT: add a0, sp, a0 +; RV32-NEXT: sw a1, 276(a0) +; RV32-NEXT: lui a0, 24414 +; RV32-NEXT: add a0, sp, a0 +; RV32-NEXT: sw a2, 280(a0) +; RV32-NEXT: lui a0, 24414 +; RV32-NEXT: add a0, sp, a0 +; RV32-NEXT: sw a3, 284(a0) +; RV32-NEXT: lui a0, 24414 +; RV32-NEXT: add a0, sp, a0 +; RV32-NEXT: sw a4, 288(a0) +; RV32-NEXT: lui a0, 24414 +; RV32-NEXT: addi a0, a0, 276 +; RV32-NEXT: add a0, sp, a0 +; RV32-NEXT: sw a0, 12(sp) +; RV32-NEXT: lw a0, 12(sp) +; RV32-NEXT: lui a1, 24414 +; RV32-NEXT: add a1, sp, a1 +; RV32-NEXT: sw a5, 292(a1) +; RV32-NEXT: lui a1, 24414 +; RV32-NEXT: add a1, sp, a1 +; RV32-NEXT: sw a6, 296(a1) +; RV32-NEXT: lui a1, 24414 +; RV32-NEXT: add a1, sp, a1 +; RV32-NEXT: sw a7, 300(a1) +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, 12(sp) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: lui a1, 24414 +; RV32-NEXT: addi a1, a1, 304 +; RV32-NEXT: add sp, sp, a1 +; RV32-NEXT: ret +; +; RV64-LABEL: va_large_stack: +; RV64: # %bb.0: +; RV64-NEXT: lui a0, 24414 +; RV64-NEXT: addiw a0, a0, 336 +; RV64-NEXT: sub sp, sp, a0 +; RV64-NEXT: .cfi_def_cfa_offset 100000080 +; RV64-NEXT: lui a0, 24414 +; RV64-NEXT: add a0, sp, a0 +; RV64-NEXT: sd a1, 280(a0) +; RV64-NEXT: lui a0, 24414 +; RV64-NEXT: add a0, sp, a0 +; RV64-NEXT: sd a2, 288(a0) +; RV64-NEXT: lui a0, 24414 +; RV64-NEXT: add a0, sp, a0 +; RV64-NEXT: sd a3, 296(a0) +; RV64-NEXT: lui a0, 24414 +; RV64-NEXT: add a0, sp, a0 +; RV64-NEXT: sd a4, 304(a0) +; RV64-NEXT: lui a0, 24414 +; RV64-NEXT: add a0, sp, a0 +; RV64-NEXT: sd a5, 312(a0) +; RV64-NEXT: lui a0, 24414 +; RV64-NEXT: addiw a0, a0, 280 +; RV64-NEXT: add a0, sp, a0 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: lw a0, 12(sp) +; RV64-NEXT: lwu a1, 8(sp) +; RV64-NEXT: lui a2, 24414 +; RV64-NEXT: add a2, sp, a2 +; RV64-NEXT: sd a6, 320(a2) +; RV64-NEXT: lui a2, 24414 +; RV64-NEXT: add a2, sp, a2 +; RV64-NEXT: sd a7, 328(a2) +; RV64-NEXT: slli a0, a0, 32 +; RV64-NEXT: or a0, a0, a1 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: srli a2, a1, 32 +; RV64-NEXT: sw a1, 8(sp) +; RV64-NEXT: sw a2, 12(sp) +; RV64-NEXT: lw a0, 0(a0) +; RV64-NEXT: lui a1, 24414 +; RV64-NEXT: addiw a1, a1, 336 +; RV64-NEXT: add sp, sp, a1 +; RV64-NEXT: ret + %large = alloca [ 100000000 x i8 ] + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load ptr, ptr %va, align 4 + %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4 + store ptr %argp.next, ptr %va, align 4 + %1 = load i32, ptr %argp.cur, align 4 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + + + diff --git a/llvm/test/CodeGen/RISCV/llvm.frexp.ll b/llvm/test/CodeGen/RISCV/llvm.frexp.ll index f8a8cfc60f14d69f024a2adc1ca133df20e6bdca..94b9444dfaf8c5b7196299501d8337803b1ab865 100644 --- a/llvm/test/CodeGen/RISCV/llvm.frexp.ll +++ b/llvm/test/CodeGen/RISCV/llvm.frexp.ll @@ -18,24 +18,279 @@ ; TODO: FIXMEs are copied blindly across from the X86 version of this test. -; FIXME -; define { half, i32 } @test_frexp_f16_i32(half %a) nounwind { -; %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) -; ret { half, i32 } %result -; } +define { half, i32 } @test_frexp_f16_i32(half %a) nounwind { +; RV32IFD-LABEL: test_frexp_f16_i32: +; RV32IFD: # %bb.0: +; RV32IFD-NEXT: addi sp, sp, -16 +; RV32IFD-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: call __extendhfsf2@plt +; RV32IFD-NEXT: addi a0, sp, 8 +; RV32IFD-NEXT: call frexpf@plt +; RV32IFD-NEXT: call __truncsfhf2@plt +; RV32IFD-NEXT: fmv.x.w a1, fa0 +; RV32IFD-NEXT: lw a0, 8(sp) +; RV32IFD-NEXT: lui a2, 1048560 +; RV32IFD-NEXT: or a1, a1, a2 +; RV32IFD-NEXT: fmv.w.x fa0, a1 +; RV32IFD-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: addi sp, sp, 16 +; RV32IFD-NEXT: ret +; +; RV64IFD-LABEL: test_frexp_f16_i32: +; RV64IFD: # %bb.0: +; RV64IFD-NEXT: addi sp, sp, -16 +; RV64IFD-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64IFD-NEXT: call __extendhfsf2@plt +; RV64IFD-NEXT: mv a0, sp +; RV64IFD-NEXT: call frexpf@plt +; RV64IFD-NEXT: call __truncsfhf2@plt +; RV64IFD-NEXT: fmv.x.w a1, fa0 +; RV64IFD-NEXT: ld a0, 0(sp) +; RV64IFD-NEXT: lui a2, 1048560 +; RV64IFD-NEXT: or a1, a1, a2 +; RV64IFD-NEXT: fmv.w.x fa0, a1 +; RV64IFD-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64IFD-NEXT: addi sp, sp, 16 +; RV64IFD-NEXT: ret +; +; RV32IZFINXZDINX-LABEL: test_frexp_f16_i32: +; RV32IZFINXZDINX: # %bb.0: +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: call __extendhfsf2@plt +; RV32IZFINXZDINX-NEXT: addi a1, sp, 8 +; RV32IZFINXZDINX-NEXT: call frexpf@plt +; RV32IZFINXZDINX-NEXT: call __truncsfhf2@plt +; RV32IZFINXZDINX-NEXT: lw a1, 8(sp) +; RV32IZFINXZDINX-NEXT: lui a2, 1048560 +; RV32IZFINXZDINX-NEXT: or a0, a0, a2 +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: ret +; +; RV64IZFINXZDINX-LABEL: test_frexp_f16_i32: +; RV64IZFINXZDINX: # %bb.0: +; RV64IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV64IZFINXZDINX-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64IZFINXZDINX-NEXT: call __extendhfsf2@plt +; RV64IZFINXZDINX-NEXT: mv a1, sp +; RV64IZFINXZDINX-NEXT: call frexpf@plt +; RV64IZFINXZDINX-NEXT: call __truncsfhf2@plt +; RV64IZFINXZDINX-NEXT: ld a1, 0(sp) +; RV64IZFINXZDINX-NEXT: lui a2, 1048560 +; RV64IZFINXZDINX-NEXT: or a0, a0, a2 +; RV64IZFINXZDINX-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV64IZFINXZDINX-NEXT: ret +; +; RV32I-LABEL: test_frexp_f16_i32: +; RV32I: # %bb.0: +; RV32I-NEXT: addi sp, sp, -16 +; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32I-NEXT: slli a0, a0, 16 +; RV32I-NEXT: srli a0, a0, 16 +; RV32I-NEXT: call __extendhfsf2@plt +; RV32I-NEXT: addi a1, sp, 8 +; RV32I-NEXT: call frexpf@plt +; RV32I-NEXT: call __truncsfhf2@plt +; RV32I-NEXT: lw a1, 8(sp) +; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32I-NEXT: addi sp, sp, 16 +; RV32I-NEXT: ret +; +; RV64I-LABEL: test_frexp_f16_i32: +; RV64I: # %bb.0: +; RV64I-NEXT: addi sp, sp, -16 +; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64I-NEXT: slli a0, a0, 48 +; RV64I-NEXT: srli a0, a0, 48 +; RV64I-NEXT: call __extendhfsf2@plt +; RV64I-NEXT: addi a1, sp, 4 +; RV64I-NEXT: call frexpf@plt +; RV64I-NEXT: call __truncsfhf2@plt +; RV64I-NEXT: lw a1, 4(sp) +; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64I-NEXT: addi sp, sp, 16 +; RV64I-NEXT: ret + %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) + ret { half, i32 } %result +} -; define half @test_frexp_f16_i32_only_use_fract(half %a) nounwind { -; %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) -; %result.0 = extractvalue { half, i32 } %result, 0 -; ret half %result.0 -; } +define half @test_frexp_f16_i32_only_use_fract(half %a) nounwind { +; RV32IFD-LABEL: test_frexp_f16_i32_only_use_fract: +; RV32IFD: # %bb.0: +; RV32IFD-NEXT: addi sp, sp, -16 +; RV32IFD-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: call __extendhfsf2@plt +; RV32IFD-NEXT: addi a0, sp, 8 +; RV32IFD-NEXT: call frexpf@plt +; RV32IFD-NEXT: call __truncsfhf2@plt +; RV32IFD-NEXT: fmv.x.w a0, fa0 +; RV32IFD-NEXT: lui a1, 1048560 +; RV32IFD-NEXT: or a0, a0, a1 +; RV32IFD-NEXT: fmv.w.x fa0, a0 +; RV32IFD-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: addi sp, sp, 16 +; RV32IFD-NEXT: ret +; +; RV64IFD-LABEL: test_frexp_f16_i32_only_use_fract: +; RV64IFD: # %bb.0: +; RV64IFD-NEXT: addi sp, sp, -16 +; RV64IFD-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64IFD-NEXT: call __extendhfsf2@plt +; RV64IFD-NEXT: mv a0, sp +; RV64IFD-NEXT: call frexpf@plt +; RV64IFD-NEXT: call __truncsfhf2@plt +; RV64IFD-NEXT: fmv.x.w a0, fa0 +; RV64IFD-NEXT: lui a1, 1048560 +; RV64IFD-NEXT: or a0, a0, a1 +; RV64IFD-NEXT: fmv.w.x fa0, a0 +; RV64IFD-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64IFD-NEXT: addi sp, sp, 16 +; RV64IFD-NEXT: ret +; +; RV32IZFINXZDINX-LABEL: test_frexp_f16_i32_only_use_fract: +; RV32IZFINXZDINX: # %bb.0: +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: call __extendhfsf2@plt +; RV32IZFINXZDINX-NEXT: addi a1, sp, 8 +; RV32IZFINXZDINX-NEXT: call frexpf@plt +; RV32IZFINXZDINX-NEXT: call __truncsfhf2@plt +; RV32IZFINXZDINX-NEXT: lui a1, 1048560 +; RV32IZFINXZDINX-NEXT: or a0, a0, a1 +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: ret +; +; RV64IZFINXZDINX-LABEL: test_frexp_f16_i32_only_use_fract: +; RV64IZFINXZDINX: # %bb.0: +; RV64IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV64IZFINXZDINX-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64IZFINXZDINX-NEXT: call __extendhfsf2@plt +; RV64IZFINXZDINX-NEXT: mv a1, sp +; RV64IZFINXZDINX-NEXT: call frexpf@plt +; RV64IZFINXZDINX-NEXT: call __truncsfhf2@plt +; RV64IZFINXZDINX-NEXT: lui a1, 1048560 +; RV64IZFINXZDINX-NEXT: or a0, a0, a1 +; RV64IZFINXZDINX-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV64IZFINXZDINX-NEXT: ret +; +; RV32I-LABEL: test_frexp_f16_i32_only_use_fract: +; RV32I: # %bb.0: +; RV32I-NEXT: addi sp, sp, -16 +; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32I-NEXT: slli a0, a0, 16 +; RV32I-NEXT: srli a0, a0, 16 +; RV32I-NEXT: call __extendhfsf2@plt +; RV32I-NEXT: addi a1, sp, 8 +; RV32I-NEXT: call frexpf@plt +; RV32I-NEXT: call __truncsfhf2@plt +; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32I-NEXT: addi sp, sp, 16 +; RV32I-NEXT: ret +; +; RV64I-LABEL: test_frexp_f16_i32_only_use_fract: +; RV64I: # %bb.0: +; RV64I-NEXT: addi sp, sp, -16 +; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64I-NEXT: slli a0, a0, 48 +; RV64I-NEXT: srli a0, a0, 48 +; RV64I-NEXT: call __extendhfsf2@plt +; RV64I-NEXT: addi a1, sp, 4 +; RV64I-NEXT: call frexpf@plt +; RV64I-NEXT: call __truncsfhf2@plt +; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64I-NEXT: addi sp, sp, 16 +; RV64I-NEXT: ret + %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) + %result.0 = extractvalue { half, i32 } %result, 0 + ret half %result.0 +} -; define i32 @test_frexp_f16_i32_only_use_exp(half %a) nounwind { -; %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) -; %result.0 = extractvalue { half, i32 } %result, 1 -; ret i32 %result.0 -; } +define i32 @test_frexp_f16_i32_only_use_exp(half %a) nounwind { +; RV32IFD-LABEL: test_frexp_f16_i32_only_use_exp: +; RV32IFD: # %bb.0: +; RV32IFD-NEXT: addi sp, sp, -16 +; RV32IFD-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: call __extendhfsf2@plt +; RV32IFD-NEXT: addi a0, sp, 8 +; RV32IFD-NEXT: call frexpf@plt +; RV32IFD-NEXT: lw a0, 8(sp) +; RV32IFD-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: addi sp, sp, 16 +; RV32IFD-NEXT: ret +; +; RV64IFD-LABEL: test_frexp_f16_i32_only_use_exp: +; RV64IFD: # %bb.0: +; RV64IFD-NEXT: addi sp, sp, -16 +; RV64IFD-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64IFD-NEXT: call __extendhfsf2@plt +; RV64IFD-NEXT: mv a0, sp +; RV64IFD-NEXT: call frexpf@plt +; RV64IFD-NEXT: ld a0, 0(sp) +; RV64IFD-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64IFD-NEXT: addi sp, sp, 16 +; RV64IFD-NEXT: ret +; +; RV32IZFINXZDINX-LABEL: test_frexp_f16_i32_only_use_exp: +; RV32IZFINXZDINX: # %bb.0: +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: call __extendhfsf2@plt +; RV32IZFINXZDINX-NEXT: addi a1, sp, 8 +; RV32IZFINXZDINX-NEXT: call frexpf@plt +; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: ret +; +; RV64IZFINXZDINX-LABEL: test_frexp_f16_i32_only_use_exp: +; RV64IZFINXZDINX: # %bb.0: +; RV64IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV64IZFINXZDINX-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64IZFINXZDINX-NEXT: call __extendhfsf2@plt +; RV64IZFINXZDINX-NEXT: mv a1, sp +; RV64IZFINXZDINX-NEXT: call frexpf@plt +; RV64IZFINXZDINX-NEXT: ld a0, 0(sp) +; RV64IZFINXZDINX-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV64IZFINXZDINX-NEXT: ret +; +; RV32I-LABEL: test_frexp_f16_i32_only_use_exp: +; RV32I: # %bb.0: +; RV32I-NEXT: addi sp, sp, -16 +; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32I-NEXT: slli a0, a0, 16 +; RV32I-NEXT: srli a0, a0, 16 +; RV32I-NEXT: call __extendhfsf2@plt +; RV32I-NEXT: addi a1, sp, 8 +; RV32I-NEXT: call frexpf@plt +; RV32I-NEXT: lw a0, 8(sp) +; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32I-NEXT: addi sp, sp, 16 +; RV32I-NEXT: ret +; +; RV64I-LABEL: test_frexp_f16_i32_only_use_exp: +; RV64I: # %bb.0: +; RV64I-NEXT: addi sp, sp, -16 +; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64I-NEXT: slli a0, a0, 48 +; RV64I-NEXT: srli a0, a0, 48 +; RV64I-NEXT: call __extendhfsf2@plt +; RV64I-NEXT: addi a1, sp, 4 +; RV64I-NEXT: call frexpf@plt +; RV64I-NEXT: lw a0, 4(sp) +; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64I-NEXT: addi sp, sp, 16 +; RV64I-NEXT: ret + %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) + %result.0 = extractvalue { half, i32 } %result, 1 + ret i32 %result.0 +} +; FIXME ; define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) nounwind { ; %result = call { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half> %a) ; ret { <2 x half>, <2 x i32> } %result diff --git a/llvm/test/CodeGen/RISCV/macro-fusions-veyron-v1.mir b/llvm/test/CodeGen/RISCV/macro-fusions-veyron-v1.mir new file mode 100644 index 0000000000000000000000000000000000000000..6d1e92e997b324c1aa70b5d125b97be7066fbaa5 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/macro-fusions-veyron-v1.mir @@ -0,0 +1,159 @@ +# REQUIRES: asserts +# RUN: llc -mtriple=riscv64-linux-gnu -mcpu=veyron-v1 -x=mir < %s \ +# RUN: -debug-only=machine-scheduler -start-before=machine-scheduler 2>&1 \ +# RUN: -mattr=+lui-addi-fusion,+auipc-addi-fusion,+shifted-zext-fusion,+ld-add-fusion \ +# RUN: | FileCheck %s + +# CHECK: lui_addi:%bb.0 +# CHECK: Macro fuse: {{.*}}LUI - ADDI +--- +name: lui_addi +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = LUI 1 + %3:gpr = XORI %1, 2 + %4:gpr = ADDI %2, 3 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: auipc_addi +# CHECK: Macro fuse: {{.*}}AUIPC - ADDI +--- +name: auipc_addi +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = AUIPC 1 + %3:gpr = XORI %1, 2 + %4:gpr = ADDI %2, 3 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: slli_srli +# CHECK: Macro fuse: {{.*}}SLLI - SRLI +--- +name: slli_srli +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = SLLI %1, 32 + %3:gpr = XORI %1, 3 + %4:gpr = SRLI %2, 4 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: slli_srli_48 +# CHECK: Macro fuse: {{.*}}SLLI - SRLI +--- +name: slli_srli_48 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = SLLI %1, 48 + %3:gpr = XORI %1, 3 + %4:gpr = SRLI %2, 48 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: slli_srli_no_fusion_0 +# CHECK-NOT: Macro fuse: {{.*}}SLLI - SRLI +--- +name: slli_srli_no_fusion_0 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = SLLI %1, 32 + %3:gpr = XORI %1, 3 + %4:gpr = SRLI %2, 33 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: slli_srli_no_fusion_1 +# CHECK-NOT: Macro fuse: {{.*}}SLLI - SRLI +--- +name: slli_srli_no_fusion_1 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = SLLI %1, 48 + %3:gpr = XORI %1, 3 + %4:gpr = SRLI %2, 4 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: slli_srli_no_fusion_2 +# CHECK-NOT: Macro fuse: {{.*}}SLLI - SRLI +--- +name: slli_srli_no_fusion_2 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = SLLI %1, 31 + %3:gpr = XORI %1, 3 + %4:gpr = SRLI %2, 4 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: slli_srli_no_fusion_3 +# CHECK-NOT: Macro fuse: {{.*}}SLLI - SRLI +--- +name: slli_srli_no_fusion_3 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = SLLI %1, 31 + %3:gpr = XORI %1, 3 + %4:gpr = SRLI %2, 48 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: ld_add +# CHECK: Macro fuse: {{.*}}ADD - LD +--- +name: ld_add +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10, $x11 + %1:gpr = COPY $x10 + %2:gpr = COPY $x11 + %3:gpr = ADD %1, %2 + %4:gpr = XORI %2, 3 + %5:gpr = LD %3, 0 + $x10 = COPY %4 + $x11 = COPY %5 + PseudoRET +... diff --git a/llvm/test/CodeGen/RISCV/relax-per-target-feature.ll b/llvm/test/CodeGen/RISCV/relax-per-target-feature.ll new file mode 100644 index 0000000000000000000000000000000000000000..7e94965cc59bd7149e64c8668acd2b54d0952cd8 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/relax-per-target-feature.ll @@ -0,0 +1,34 @@ +; RUN: llc -filetype=obj -mtriple=riscv64 %s -o %t +; RUN: llvm-objdump -dr -M no-aliases --mattr=+c %t | FileCheck %s + +;; Functions may have more features than the base triple; code generation and +;; instruction selection may be performed based on this information. This test +;; makes sure that the MC layer uses the target-features of the function. + +declare dso_local i32 @ext(i32) + +; CHECK-LABEL: : +; CHECK-NEXT: c.li a0, 31 +; CHECK-NEXT: auipc t1, 0 +; CHECK-NEXT: R_RISCV_CALL_PLT ext +; CHECK-NEXT: R_RISCV_RELAX *ABS* +; CHECK-NEXT: jalr zero, 0(t1) +define dso_local i32 @f() #0 { +entry: + %r = tail call i32 @ext(i32 31) + ret i32 %r +} + +; CHECK-LABEL: : +; CHECK-NEXT: addi a0, zero, 31 +; CHECK-NEXT: auipc t1, 0 +; CHECK-NEXT: R_RISCV_CALL_PLT ext +; CHECK-NEXT: jalr zero, 0(t1) +define dso_local i32 @g() #1 { +entry: + %r = tail call i32 @ext(i32 31) + ret i32 %r +} + +attributes #0 = { nounwind "target-features"="+c,+relax" } +attributes #1 = { nounwind "target-features"="-c,-relax" } diff --git a/llvm/test/CodeGen/RISCV/riscv-codegenprepare-asm.ll b/llvm/test/CodeGen/RISCV/riscv-codegenprepare-asm.ll index e73249739f356bc3795db2b8fa9d5d7f16255171..31ca8eab33508c7e94fc416a5e35c9cc762348db 100644 --- a/llvm/test/CodeGen/RISCV/riscv-codegenprepare-asm.ll +++ b/llvm/test/CodeGen/RISCV/riscv-codegenprepare-asm.ll @@ -10,8 +10,7 @@ define void @test1(ptr nocapture noundef %a, i32 noundef signext %n) { ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: blez a1, .LBB0_3 ; CHECK-NEXT: # %bb.1: # %for.body.preheader -; CHECK-NEXT: slli a1, a1, 32 -; CHECK-NEXT: srli a1, a1, 30 +; CHECK-NEXT: slli a1, a1, 2 ; CHECK-NEXT: add a1, a0, a1 ; CHECK-NEXT: .LBB0_2: # %for.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll index a56a81f5f793bc2de7f48dabba8c86b2d29d46a1..769bfe8cd5ba97cb0afd97f306acef4d55dca723 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll @@ -244,14 +244,14 @@ define <8 x i64> @vrgather_shuffle_vx_v8i64(<8 x i64> %x) { ; RV32-NEXT: addi a0, a0, %lo(.LCPI13_0) ; RV32-NEXT: vsetivli zero, 8, e64, m4, ta, mu ; RV32-NEXT: vle16.v v16, (a0) -; RV32-NEXT: vrgatherei16.vv v12, v8, v16 +; RV32-NEXT: vmv.v.i v20, 5 ; RV32-NEXT: lui a0, %hi(.LCPI13_1) ; RV32-NEXT: addi a0, a0, %lo(.LCPI13_1) -; RV32-NEXT: vle16.v v8, (a0) -; RV32-NEXT: li a0, 140 +; RV32-NEXT: vle16.v v17, (a0) +; RV32-NEXT: li a0, 115 ; RV32-NEXT: vmv.s.x v0, a0 -; RV32-NEXT: vmv.v.i v16, 5 -; RV32-NEXT: vrgatherei16.vv v12, v16, v8, v0.t +; RV32-NEXT: vrgatherei16.vv v12, v20, v16 +; RV32-NEXT: vrgatherei16.vv v12, v8, v17, v0.t ; RV32-NEXT: vmv.v.v v8, v12 ; RV32-NEXT: ret ; diff --git a/llvm/test/CodeGen/RISCV/sextw-removal.ll b/llvm/test/CodeGen/RISCV/sextw-removal.ll index 3babef93499c85943b208aa924f67dd386b7de21..e730092674e06a258529fe9eb5367b85a7fa84f2 100644 --- a/llvm/test/CodeGen/RISCV/sextw-removal.ll +++ b/llvm/test/CodeGen/RISCV/sextw-removal.ll @@ -1,9 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+f,+zknh -target-abi=lp64f \ +; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+f,+zknh,+v -target-abi=lp64f \ ; RUN: | FileCheck %s --check-prefixes=CHECK,RV64I -; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+zba,+zbb,+f,+zknh -target-abi=lp64f \ +; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+zba,+zbb,+f,+zknh,+v -target-abi=lp64f \ ; RUN: | FileCheck %s --check-prefixes=CHECK,RV64ZBB -; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+zba,+zbb,+f,+zknh -target-abi=lp64f \ +; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+zba,+zbb,+f,+zknh,+v -target-abi=lp64f \ ; RUN: -riscv-disable-sextw-removal | FileCheck %s --check-prefix=NOREMOVAL define void @test1(i32 signext %arg, i32 signext %arg1) nounwind { @@ -1436,3 +1436,64 @@ bb7: ; preds = %bb2 } declare void @side_effect(i64) + +define void @test20( %arg, i32 signext %arg1) nounwind { +; CHECK-LABEL: test20: +; CHECK: # %bb.0: # %bb +; CHECK-NEXT: addi sp, sp, -32 +; CHECK-NEXT: sd ra, 24(sp) # 8-byte Folded Spill +; CHECK-NEXT: sd s0, 16(sp) # 8-byte Folded Spill +; CHECK-NEXT: sd s1, 8(sp) # 8-byte Folded Spill +; CHECK-NEXT: mv s0, a0 +; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma +; CHECK-NEXT: vmv.x.s s1, v8 +; CHECK-NEXT: .LBB24_1: # %bb2 +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: mv a0, s1 +; CHECK-NEXT: call bar@plt +; CHECK-NEXT: sllw s1, s1, s0 +; CHECK-NEXT: bnez a0, .LBB24_1 +; CHECK-NEXT: # %bb.2: # %bb7 +; CHECK-NEXT: ld ra, 24(sp) # 8-byte Folded Reload +; CHECK-NEXT: ld s0, 16(sp) # 8-byte Folded Reload +; CHECK-NEXT: ld s1, 8(sp) # 8-byte Folded Reload +; CHECK-NEXT: addi sp, sp, 32 +; CHECK-NEXT: ret +; +; NOREMOVAL-LABEL: test20: +; NOREMOVAL: # %bb.0: # %bb +; NOREMOVAL-NEXT: addi sp, sp, -32 +; NOREMOVAL-NEXT: sd ra, 24(sp) # 8-byte Folded Spill +; NOREMOVAL-NEXT: sd s0, 16(sp) # 8-byte Folded Spill +; NOREMOVAL-NEXT: sd s1, 8(sp) # 8-byte Folded Spill +; NOREMOVAL-NEXT: mv s0, a0 +; NOREMOVAL-NEXT: vsetivli zero, 1, e32, mf2, ta, ma +; NOREMOVAL-NEXT: vmv.x.s s1, v8 +; NOREMOVAL-NEXT: .LBB24_1: # %bb2 +; NOREMOVAL-NEXT: # =>This Inner Loop Header: Depth=1 +; NOREMOVAL-NEXT: sext.w a0, s1 +; NOREMOVAL-NEXT: call bar@plt +; NOREMOVAL-NEXT: sllw s1, s1, s0 +; NOREMOVAL-NEXT: bnez a0, .LBB24_1 +; NOREMOVAL-NEXT: # %bb.2: # %bb7 +; NOREMOVAL-NEXT: ld ra, 24(sp) # 8-byte Folded Reload +; NOREMOVAL-NEXT: ld s0, 16(sp) # 8-byte Folded Reload +; NOREMOVAL-NEXT: ld s1, 8(sp) # 8-byte Folded Reload +; NOREMOVAL-NEXT: addi sp, sp, 32 +; NOREMOVAL-NEXT: ret +bb: + %i = call i32 @llvm.riscv.vmv.x.s.nxv1i32( %arg) + br label %bb2 + +bb2: ; preds = %bb2, %bb + %i3 = phi i32 [ %i, %bb ], [ %i5, %bb2 ] + %i4 = tail call signext i32 @bar(i32 signext %i3) + %i5 = shl i32 %i3, %arg1 + %i6 = icmp eq i32 %i4, 0 + br i1 %i6, label %bb7, label %bb2 + +bb7: ; preds = %bb2 + ret void +} + +declare i32 @llvm.riscv.vmv.x.s.nxv1i32( ) diff --git a/llvm/test/CodeGen/SPIRV/opencl/basic/get_global_offset.ll b/llvm/test/CodeGen/SPIRV/opencl/basic/get_global_offset.ll index da6015d23d0549aaeb8bc696c3c69c9a5769ed3f..127804671cee475102b8d3fa1f08970ab2317710 100644 --- a/llvm/test/CodeGen/SPIRV/opencl/basic/get_global_offset.ll +++ b/llvm/test/CodeGen/SPIRV/opencl/basic/get_global_offset.ll @@ -12,6 +12,8 @@ ; CHECK: %[[#int64_ty:]] = OpTypeInt 64 0 ; CHECK: %[[#vec_ty:]] = OpTypeVector %[[#int64_ty]] 3 ; CHECK: %[[#func2_ty:]] = OpTypeFunction %[[#vec_ty]] +; CHECK: %[[#int32_ty:]] = OpTypeInt 32 0 +; CHECK: %[[#i32ptr_ty:]] = OpTypePointer CrossWorkgroup %[[#int32_ty]] ;; TODO: add 64-bit constant defs ; CHECK: %[[#f2_decl]] = OpFunction %[[#vec_ty]] Pure %[[#func2_ty]] ; CHECK: OpFunctionEnd @@ -31,21 +33,21 @@ entry: %0 = call spir_func <3 x i64> @BuiltInGlobalOffset() #1 %call = extractelement <3 x i64> %0, i32 0 %conv = trunc i64 %call to i32 -; CHECK: %[[#i1:]] = OpInBoundsPtrAccessChain %[[#iptr_ty]] %[[#outOffsets]] +; CHECK: %[[#i1:]] = OpInBoundsPtrAccessChain %[[#i32ptr_ty]] %[[#outOffsets]] ; CHECK: OpStore %[[#i1:]] %[[#]] Aligned 4 %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %outOffsets, i64 0 store i32 %conv, i32 addrspace(1)* %arrayidx, align 4 %1 = call spir_func <3 x i64> @BuiltInGlobalOffset() #1 %call1 = extractelement <3 x i64> %1, i32 1 %conv2 = trunc i64 %call1 to i32 -; CHECK: %[[#i2:]] = OpInBoundsPtrAccessChain %[[#iptr_ty]] %[[#outOffsets]] +; CHECK: %[[#i2:]] = OpInBoundsPtrAccessChain %[[#i32ptr_ty]] %[[#outOffsets]] ; CHECK: OpStore %[[#i2:]] %[[#]] Aligned 4 %arrayidx3 = getelementptr inbounds i32, i32 addrspace(1)* %outOffsets, i64 1 store i32 %conv2, i32 addrspace(1)* %arrayidx3, align 4 %2 = call spir_func <3 x i64> @BuiltInGlobalOffset() #1 %call4 = extractelement <3 x i64> %2, i32 2 %conv5 = trunc i64 %call4 to i32 -; CHECK: %[[#i3:]] = OpInBoundsPtrAccessChain %[[#iptr_ty]] %[[#outOffsets]] +; CHECK: %[[#i3:]] = OpInBoundsPtrAccessChain %[[#i32ptr_ty]] %[[#outOffsets]] ; CHECK: OpStore %[[#i3:]] %[[#]] Aligned 4 %arrayidx6 = getelementptr inbounds i32, i32 addrspace(1)* %outOffsets, i64 2 store i32 %conv5, i32 addrspace(1)* %arrayidx6, align 4 diff --git a/llvm/test/CodeGen/SPIRV/struct.ll b/llvm/test/CodeGen/SPIRV/struct.ll index 3c4fc81b920045738a5ddf24806a89f6135c8b51..c3d2694cf929cc15c7a303e17a3aaf96d3a42386 100644 --- a/llvm/test/CodeGen/SPIRV/struct.ll +++ b/llvm/test/CodeGen/SPIRV/struct.ll @@ -3,11 +3,10 @@ %struct.ST = type { i32, i32, i32 } ; CHECK-DAG: OpName %[[#struct:]] "struct.ST" -; CHECK-DAG: %[[#char:]] = OpTypeInt 8 0 ; CHECK-DAG: %[[#int:]] = OpTypeInt 32 0 ; CHECK-DAG: %[[#struct]] = OpTypeStruct %[[#int]] %[[#int]] %[[#int]] ; CHECK-DAG: %[[#structP:]] = OpTypePointer Function %[[#struct]] -; CHECK-DAG: %[[#intP:]] = OpTypePointer Function %[[#char]] +; CHECK-DAG: %[[#intP:]] = OpTypePointer Function %[[#int]] ; CHECK-DAG: %[[#zero:]] = OpConstant %[[#int]] 0 ; CHECK-DAG: %[[#one:]] = OpConstant %[[#int]] 1 ; CHECK-DAG: %[[#two:]] = OpConstant %[[#int]] 2 diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-add-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-add-04.ll index 50d3eec15dbe87986005b0288d78d2e2f909aaea..4af3793191b67da37dcaf60584bb2ae5cbecbbe5 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-add-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-add-04.ll @@ -16,13 +16,12 @@ define i64 @f1(i64 %dummy, ptr %src, i64 %b) { ret i64 %res } -; Check addition of 1, which can use AGHI. +; Check addition of 1. define i64 @f2(i64 %dummy, ptr %src) { ; CHECK-LABEL: f2: ; CHECK: lg %r2, 0(%r3) ; CHECK: [[LABEL:\.[^:]*]]: -; CHECK: lgr %r0, %r2 -; CHECK: aghi %r0, 1 +; CHECK: la %r0, 1(%r2) ; CHECK: csg %r2, %r0, 0(%r3) ; CHECK: jl [[LABEL]] ; CHECK: br %r14 @@ -30,82 +29,64 @@ define i64 @f2(i64 %dummy, ptr %src) { ret i64 %res } -; Check the high end of the AGHI range. +; Check use of LAY. define i64 @f3(i64 %dummy, ptr %src) { ; CHECK-LABEL: f3: -; CHECK: aghi %r0, 32767 +; CHECK: lay %r0, 32767(%r2) ; CHECK: br %r14 %res = atomicrmw add ptr %src, i64 32767 seq_cst ret i64 %res } -; Check the next value up, which must use AGFI. +; Check the high end of the AGFI range. define i64 @f4(i64 %dummy, ptr %src) { ; CHECK-LABEL: f4: -; CHECK: agfi %r0, 32768 +; CHECK: agfi %r0, 2147483647 ; CHECK: br %r14 - %res = atomicrmw add ptr %src, i64 32768 seq_cst + %res = atomicrmw add ptr %src, i64 2147483647 seq_cst ret i64 %res } -; Check the high end of the AGFI range. +; Check the next value up, which uses an ALGFI. define i64 @f5(i64 %dummy, ptr %src) { ; CHECK-LABEL: f5: -; CHECK: agfi %r0, 2147483647 +; CHECK: algfi %r0, 2147483648 ; CHECK: br %r14 - %res = atomicrmw add ptr %src, i64 2147483647 seq_cst + %res = atomicrmw add ptr %src, i64 2147483648 seq_cst ret i64 %res } -; Check the next value up, which must use a register addition. +; Check addition of -1, which can use LAY. define i64 @f6(i64 %dummy, ptr %src) { ; CHECK-LABEL: f6: -; CHECK: agr +; CHECK: lay %r0, -1(%r2) ; CHECK: br %r14 - %res = atomicrmw add ptr %src, i64 2147483648 seq_cst + %res = atomicrmw add ptr %src, i64 -1 seq_cst ret i64 %res } -; Check addition of -1, which can use AGHI. +; LAY still OK. define i64 @f7(i64 %dummy, ptr %src) { ; CHECK-LABEL: f7: -; CHECK: aghi %r0, -1 -; CHECK: br %r14 - %res = atomicrmw add ptr %src, i64 -1 seq_cst - ret i64 %res -} - -; Check the low end of the AGHI range. -define i64 @f8(i64 %dummy, ptr %src) { -; CHECK-LABEL: f8: -; CHECK: aghi %r0, -32768 -; CHECK: br %r14 - %res = atomicrmw add ptr %src, i64 -32768 seq_cst - ret i64 %res -} - -; Check the next value down, which must use AGFI instead. -define i64 @f9(i64 %dummy, ptr %src) { -; CHECK-LABEL: f9: -; CHECK: agfi %r0, -32769 +; CHECK: lay %r0, -32769(%r2) ; CHECK: br %r14 %res = atomicrmw add ptr %src, i64 -32769 seq_cst ret i64 %res } ; Check the low end of the AGFI range. -define i64 @f10(i64 %dummy, ptr %src) { -; CHECK-LABEL: f10: +define i64 @f8(i64 %dummy, ptr %src) { +; CHECK-LABEL: f8: ; CHECK: agfi %r0, -2147483648 ; CHECK: br %r14 %res = atomicrmw add ptr %src, i64 -2147483648 seq_cst ret i64 %res } -; Check the next value down, which must use a register addition. -define i64 @f11(i64 %dummy, ptr %src) { -; CHECK-LABEL: f11: -; CHECK: agr +; Check the next value down, which uses an SLGFI. +define i64 @f9(i64 %dummy, ptr %src) { +; CHECK-LABEL: f9: +; CHECK: slgfi %r0, 2147483649 ; CHECK: br %r14 %res = atomicrmw add ptr %src, i64 -2147483649 seq_cst ret i64 %res diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-and-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-and-03.ll index 03ed2404dcc17e572606e5e24d130c58e9b7fa94..96c82e6b1eaec87ed5cd78892eb646a1ab8890d8 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-and-03.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-and-03.ll @@ -33,7 +33,7 @@ define i32 @f2(i32 %dummy, ptr %src) { ; Check ANDs of the low end of the NILH range. define i32 @f3(i32 %dummy, ptr %src) { ; CHECK-LABEL: f3: -; CHECK: nilh %r0, 0 +; CHECK: llhr %r0, %r2 ; CHECK: br %r14 %res = atomicrmw and ptr %src, i32 65535 seq_cst ret i32 %res diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-and-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-and-04.ll index 00b6fd93ad5bbc1368e56b74d67e526fc5cebfaa..9647548c842bfa2b011194eb2ba4946f76e07af0 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-and-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-and-04.ll @@ -16,11 +16,10 @@ define i64 @f1(i64 %dummy, ptr %src, i64 %b) { ret i64 %res } -; Check ANDs of 1, which are done using a register. (We could use RISBG -; instead, but that isn't implemented yet.) +; Check ANDs of 1, which are done using a register. define i64 @f2(i64 %dummy, ptr %src) { ; CHECK-LABEL: f2: -; CHECK: ngr +; CHECK: risbg ; CHECK: br %r14 %res = atomicrmw and ptr %src, i64 1 seq_cst ret i64 %res @@ -56,7 +55,7 @@ define i64 @f4(i64 %dummy, ptr %src) { ; Check the next value up, which must use a register. define i64 @f5(i64 %dummy, ptr %src) { ; CHECK-LABEL: f5: -; CHECK: ngr +; CHECK: risbg ; CHECK: br %r14 %res = atomicrmw and ptr %src, i64 12884901888 seq_cst ret i64 %res @@ -74,7 +73,7 @@ define i64 @f6(i64 %dummy, ptr %src) { ; Check the next value up, which must use a register. define i64 @f7(i64 %dummy, ptr %src) { ; CHECK-LABEL: f7: -; CHECK: ngr +; CHECK: risbg ; CHECK: br %r14 %res = atomicrmw and ptr %src, i64 281474976710656 seq_cst ret i64 %res diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll index d633c2d74e3a6bd9a0e94f2a8d2514acfc7f2ff7..d107e5d1dc2e580e821e7d53576c3d1554773b1d 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll @@ -1,21 +1,31 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; Test 32-bit atomic minimum and maximum. Here we match the z10 versions, ; which can't use LOCR. ; ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s +; Todo: If-converter no longer producing CondReturns (with AtomicExpand pass). + ; Check signed minimum. define i32 @f1(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f1: -; CHECK: l %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: cs %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP]]: -; CHECK: lr [[NEW]], %r2 -; CHECK: crjle %r2, %r4, [[KEEP:\..*]] -; CHECK: lr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: l %r2, 0(%r3) +; CHECK-NEXT: j .LBB0_2 +; CHECK-NEXT: .LBB0_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB0_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB0_4 +; CHECK-NEXT: .LBB0_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB0_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB0_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB0_1 +; CHECK-NEXT: .LBB0_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw min ptr %src, i32 %b seq_cst ret i32 %res } @@ -23,16 +33,23 @@ define i32 @f1(i32 %dummy, ptr %src, i32 %b) { ; Check signed maximum. define i32 @f2(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f2: -; CHECK: l %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: cs %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP]]: -; CHECK: lr [[NEW]], %r2 -; CHECK: crjhe %r2, %r4, [[KEEP:\..*]] -; CHECK: lr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: l %r2, 0(%r3) +; CHECK-NEXT: j .LBB1_2 +; CHECK-NEXT: .LBB1_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB1_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB1_4 +; CHECK-NEXT: .LBB1_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjh %r2, %r4, .LBB1_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB1_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB1_1 +; CHECK-NEXT: .LBB1_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw max ptr %src, i32 %b seq_cst ret i32 %res } @@ -40,16 +57,23 @@ define i32 @f2(i32 %dummy, ptr %src, i32 %b) { ; Check unsigned minimum. define i32 @f3(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f3: -; CHECK: l %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: cs %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP]]: -; CHECK: lr [[NEW]], %r2 -; CHECK: clrjle %r2, %r4, [[KEEP:\..*]] -; CHECK: lr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: l %r2, 0(%r3) +; CHECK-NEXT: j .LBB2_2 +; CHECK-NEXT: .LBB2_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB2_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB2_4 +; CHECK-NEXT: .LBB2_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: clrjle %r2, %r4, .LBB2_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB2_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB2_1 +; CHECK-NEXT: .LBB2_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw umin ptr %src, i32 %b seq_cst ret i32 %res } @@ -57,16 +81,23 @@ define i32 @f3(i32 %dummy, ptr %src, i32 %b) { ; Check unsigned maximum. define i32 @f4(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f4: -; CHECK: l %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: cs %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP]]: -; CHECK: lr [[NEW]], %r2 -; CHECK: clrjhe %r2, %r4, [[KEEP:\..*]] -; CHECK: lr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: l %r2, 0(%r3) +; CHECK-NEXT: j .LBB3_2 +; CHECK-NEXT: .LBB3_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB3_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB3_4 +; CHECK-NEXT: .LBB3_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: clrjh %r2, %r4, .LBB3_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB3_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB3_1 +; CHECK-NEXT: .LBB3_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw umax ptr %src, i32 %b seq_cst ret i32 %res } @@ -74,9 +105,23 @@ define i32 @f4(i32 %dummy, ptr %src, i32 %b) { ; Check the high end of the aligned CS range. define i32 @f5(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f5: -; CHECK: l %r2, 4092(%r3) -; CHECK: cs %r2, {{%r[0-9]+}}, 4092(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: l %r2, 4092(%r3) +; CHECK-NEXT: j .LBB4_2 +; CHECK-NEXT: .LBB4_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB4_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 4092(%r3) +; CHECK-NEXT: je .LBB4_4 +; CHECK-NEXT: .LBB4_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB4_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB4_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB4_1 +; CHECK-NEXT: .LBB4_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i32, ptr %src, i64 1023 %res = atomicrmw min ptr %ptr, i32 %b seq_cst ret i32 %res @@ -85,9 +130,23 @@ define i32 @f5(i32 %dummy, ptr %src, i32 %b) { ; Check the next word up, which requires CSY. define i32 @f6(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f6: -; CHECK: ly %r2, 4096(%r3) -; CHECK: csy %r2, {{%r[0-9]+}}, 4096(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: ly %r2, 4096(%r3) +; CHECK-NEXT: j .LBB5_2 +; CHECK-NEXT: .LBB5_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB5_2 Depth=1 +; CHECK-NEXT: csy %r2, %r0, 4096(%r3) +; CHECK-NEXT: je .LBB5_4 +; CHECK-NEXT: .LBB5_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB5_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB5_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB5_1 +; CHECK-NEXT: .LBB5_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i32, ptr %src, i64 1024 %res = atomicrmw min ptr %ptr, i32 %b seq_cst ret i32 %res @@ -96,9 +155,23 @@ define i32 @f6(i32 %dummy, ptr %src, i32 %b) { ; Check the high end of the aligned CSY range. define i32 @f7(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f7: -; CHECK: ly %r2, 524284(%r3) -; CHECK: csy %r2, {{%r[0-9]+}}, 524284(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: ly %r2, 524284(%r3) +; CHECK-NEXT: j .LBB6_2 +; CHECK-NEXT: .LBB6_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB6_2 Depth=1 +; CHECK-NEXT: csy %r2, %r0, 524284(%r3) +; CHECK-NEXT: je .LBB6_4 +; CHECK-NEXT: .LBB6_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB6_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB6_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB6_1 +; CHECK-NEXT: .LBB6_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i32, ptr %src, i64 131071 %res = atomicrmw min ptr %ptr, i32 %b seq_cst ret i32 %res @@ -107,10 +180,24 @@ define i32 @f7(i32 %dummy, ptr %src, i32 %b) { ; Check the next word up, which needs separate address logic. define i32 @f8(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f8: -; CHECK: agfi %r3, 524288 -; CHECK: l %r2, 0(%r3) -; CHECK: cs %r2, {{%r[0-9]+}}, 0(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: agfi %r3, 524288 +; CHECK-NEXT: l %r2, 0(%r3) +; CHECK-NEXT: j .LBB7_2 +; CHECK-NEXT: .LBB7_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB7_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB7_4 +; CHECK-NEXT: .LBB7_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB7_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB7_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB7_1 +; CHECK-NEXT: .LBB7_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i32, ptr %src, i64 131072 %res = atomicrmw min ptr %ptr, i32 %b seq_cst ret i32 %res @@ -119,9 +206,23 @@ define i32 @f8(i32 %dummy, ptr %src, i32 %b) { ; Check the high end of the negative aligned CSY range. define i32 @f9(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f9: -; CHECK: ly %r2, -4(%r3) -; CHECK: csy %r2, {{%r[0-9]+}}, -4(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: ly %r2, -4(%r3) +; CHECK-NEXT: j .LBB8_2 +; CHECK-NEXT: .LBB8_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB8_2 Depth=1 +; CHECK-NEXT: csy %r2, %r0, -4(%r3) +; CHECK-NEXT: je .LBB8_4 +; CHECK-NEXT: .LBB8_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB8_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB8_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB8_1 +; CHECK-NEXT: .LBB8_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i32, ptr %src, i64 -1 %res = atomicrmw min ptr %ptr, i32 %b seq_cst ret i32 %res @@ -130,9 +231,23 @@ define i32 @f9(i32 %dummy, ptr %src, i32 %b) { ; Check the low end of the CSY range. define i32 @f10(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f10: -; CHECK: ly %r2, -524288(%r3) -; CHECK: csy %r2, {{%r[0-9]+}}, -524288(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: ly %r2, -524288(%r3) +; CHECK-NEXT: j .LBB9_2 +; CHECK-NEXT: .LBB9_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB9_2 Depth=1 +; CHECK-NEXT: csy %r2, %r0, -524288(%r3) +; CHECK-NEXT: je .LBB9_4 +; CHECK-NEXT: .LBB9_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB9_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB9_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB9_1 +; CHECK-NEXT: .LBB9_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i32, ptr %src, i64 -131072 %res = atomicrmw min ptr %ptr, i32 %b seq_cst ret i32 %res @@ -141,10 +256,24 @@ define i32 @f10(i32 %dummy, ptr %src, i32 %b) { ; Check the next word down, which needs separate address logic. define i32 @f11(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f11: -; CHECK: agfi %r3, -524292 -; CHECK: l %r2, 0(%r3) -; CHECK: cs %r2, {{%r[0-9]+}}, 0(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: agfi %r3, -524292 +; CHECK-NEXT: l %r2, 0(%r3) +; CHECK-NEXT: j .LBB10_2 +; CHECK-NEXT: .LBB10_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB10_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB10_4 +; CHECK-NEXT: .LBB10_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB10_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB10_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB10_1 +; CHECK-NEXT: .LBB10_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i32, ptr %src, i64 -131073 %res = atomicrmw min ptr %ptr, i32 %b seq_cst ret i32 %res @@ -153,10 +282,24 @@ define i32 @f11(i32 %dummy, ptr %src, i32 %b) { ; Check that indexed addresses are not allowed. define i32 @f12(i32 %dummy, i64 %base, i64 %index, i32 %b) { ; CHECK-LABEL: f12: -; CHECK: agr %r3, %r4 -; CHECK: l %r2, 0(%r3) -; CHECK: cs %r2, {{%r[0-9]+}}, 0(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: l %r2, 0(%r4,%r3) +; CHECK-NEXT: agr %r3, %r4 +; CHECK-NEXT: j .LBB11_2 +; CHECK-NEXT: .LBB11_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB11_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB11_4 +; CHECK-NEXT: .LBB11_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r5, .LBB11_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB11_2 Depth=1 +; CHECK-NEXT: lr %r0, %r5 +; CHECK-NEXT: j .LBB11_1 +; CHECK-NEXT: .LBB11_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %add = add i64 %base, %index %ptr = inttoptr i64 %add to ptr %res = atomicrmw min ptr %ptr, i32 %b seq_cst @@ -166,16 +309,23 @@ define i32 @f12(i32 %dummy, i64 %base, i64 %index, i32 %b) { ; Check that constants are handled. define i32 @f13(i32 %dummy, ptr %ptr) { ; CHECK-LABEL: f13: -; CHECK: lhi [[LIMIT:%r[0-9]+]], 42 -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: cs %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP]]: -; CHECK: lr [[NEW]], %r2 -; CHECK: crjle %r2, [[LIMIT]], [[KEEP:\..*]] -; CHECK: lhi [[NEW]], 42 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: l %r2, 0(%r3) +; CHECK-NEXT: j .LBB12_2 +; CHECK-NEXT: .LBB12_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB12_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB12_4 +; CHECK-NEXT: .LBB12_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: cijl %r2, 43, .LBB12_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB12_2 Depth=1 +; CHECK-NEXT: lhi %r0, 42 +; CHECK-NEXT: j .LBB12_1 +; CHECK-NEXT: .LBB12_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw min ptr %ptr, i32 42 seq_cst ret i32 %res } diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll index 64e76e0a90eaf589999b5a9c03b8575cb618573f..9352118a32f8a6dea18ebec047cf273b9c7894b7 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll @@ -1,21 +1,31 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; Test 64-bit atomic minimum and maximum. Here we match the z10 versions, ; which can't use LOCGR. ; ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s +; Todo: If-converter no longer producing CondReturns (with AtomicExpand pass). + ; Check signed minimum. define i64 @f1(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f1: -; CHECK: lg %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: csg %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP:\.[^:]*]]: -; CHECK: lgr [[NEW:%r[0-9]+]], %r2 -; CHECK: cgrjle %r2, %r4, [[KEEP:\..*]] -; CHECK: lgr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, 0(%r3) +; CHECK-NEXT: j .LBB0_2 +; CHECK-NEXT: .LBB0_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB0_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB0_4 +; CHECK-NEXT: .LBB0_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgrjle %r2, %r4, .LBB0_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB0_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB0_1 +; CHECK-NEXT: .LBB0_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw min ptr %src, i64 %b seq_cst ret i64 %res } @@ -23,16 +33,23 @@ define i64 @f1(i64 %dummy, ptr %src, i64 %b) { ; Check signed maximum. define i64 @f2(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f2: -; CHECK: lg %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: csg %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP:\.[^:]*]]: -; CHECK: lgr [[NEW:%r[0-9]+]], %r2 -; CHECK: cgrjhe %r2, %r4, [[KEEP:\..*]] -; CHECK: lgr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, 0(%r3) +; CHECK-NEXT: j .LBB1_2 +; CHECK-NEXT: .LBB1_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB1_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB1_4 +; CHECK-NEXT: .LBB1_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgrjh %r2, %r4, .LBB1_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB1_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB1_1 +; CHECK-NEXT: .LBB1_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw max ptr %src, i64 %b seq_cst ret i64 %res } @@ -40,16 +57,23 @@ define i64 @f2(i64 %dummy, ptr %src, i64 %b) { ; Check unsigned minimum. define i64 @f3(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f3: -; CHECK: lg %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: csg %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP:\.[^:]*]]: -; CHECK: lgr [[NEW:%r[0-9]+]], %r2 -; CHECK: clgrjle %r2, %r4, [[KEEP:\..*]] -; CHECK: lgr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, 0(%r3) +; CHECK-NEXT: j .LBB2_2 +; CHECK-NEXT: .LBB2_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB2_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB2_4 +; CHECK-NEXT: .LBB2_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: clgrjle %r2, %r4, .LBB2_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB2_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB2_1 +; CHECK-NEXT: .LBB2_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw umin ptr %src, i64 %b seq_cst ret i64 %res } @@ -57,16 +81,23 @@ define i64 @f3(i64 %dummy, ptr %src, i64 %b) { ; Check unsigned maximum. define i64 @f4(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f4: -; CHECK: lg %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: csg %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP:\.[^:]*]]: -; CHECK: lgr [[NEW:%r[0-9]+]], %r2 -; CHECK: clgrjhe %r2, %r4, [[KEEP:\..*]] -; CHECK: lgr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, 0(%r3) +; CHECK-NEXT: j .LBB3_2 +; CHECK-NEXT: .LBB3_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB3_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB3_4 +; CHECK-NEXT: .LBB3_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: clgrjh %r2, %r4, .LBB3_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB3_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB3_1 +; CHECK-NEXT: .LBB3_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw umax ptr %src, i64 %b seq_cst ret i64 %res } @@ -74,9 +105,23 @@ define i64 @f4(i64 %dummy, ptr %src, i64 %b) { ; Check the high end of the aligned CSG range. define i64 @f5(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f5: -; CHECK: lg %r2, 524280(%r3) -; CHECK: csg %r2, {{%r[0-9]+}}, 524280(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, 524280(%r3) +; CHECK-NEXT: j .LBB4_2 +; CHECK-NEXT: .LBB4_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB4_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 524280(%r3) +; CHECK-NEXT: je .LBB4_4 +; CHECK-NEXT: .LBB4_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgrjle %r2, %r4, .LBB4_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB4_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB4_1 +; CHECK-NEXT: .LBB4_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i64, ptr %src, i64 65535 %res = atomicrmw min ptr %ptr, i64 %b seq_cst ret i64 %res @@ -85,10 +130,24 @@ define i64 @f5(i64 %dummy, ptr %src, i64 %b) { ; Check the next doubleword up, which requires separate address logic. define i64 @f6(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f6: -; CHECK: agfi %r3, 524288 -; CHECK: lg %r2, 0(%r3) -; CHECK: csg %r2, {{%r[0-9]+}}, 0(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: agfi %r3, 524288 +; CHECK-NEXT: lg %r2, 0(%r3) +; CHECK-NEXT: j .LBB5_2 +; CHECK-NEXT: .LBB5_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB5_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB5_4 +; CHECK-NEXT: .LBB5_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgrjle %r2, %r4, .LBB5_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB5_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB5_1 +; CHECK-NEXT: .LBB5_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i64, ptr %src, i64 65536 %res = atomicrmw min ptr %ptr, i64 %b seq_cst ret i64 %res @@ -97,9 +156,23 @@ define i64 @f6(i64 %dummy, ptr %src, i64 %b) { ; Check the low end of the CSG range. define i64 @f7(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f7: -; CHECK: lg %r2, -524288(%r3) -; CHECK: csg %r2, {{%r[0-9]+}}, -524288(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, -524288(%r3) +; CHECK-NEXT: j .LBB6_2 +; CHECK-NEXT: .LBB6_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB6_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, -524288(%r3) +; CHECK-NEXT: je .LBB6_4 +; CHECK-NEXT: .LBB6_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgrjle %r2, %r4, .LBB6_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB6_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB6_1 +; CHECK-NEXT: .LBB6_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i64, ptr %src, i64 -65536 %res = atomicrmw min ptr %ptr, i64 %b seq_cst ret i64 %res @@ -108,10 +181,24 @@ define i64 @f7(i64 %dummy, ptr %src, i64 %b) { ; Check the next doubleword down, which requires separate address logic. define i64 @f8(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f8: -; CHECK: agfi %r3, -524296 -; CHECK: lg %r2, 0(%r3) -; CHECK: csg %r2, {{%r[0-9]+}}, 0(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: agfi %r3, -524296 +; CHECK-NEXT: lg %r2, 0(%r3) +; CHECK-NEXT: j .LBB7_2 +; CHECK-NEXT: .LBB7_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB7_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB7_4 +; CHECK-NEXT: .LBB7_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgrjle %r2, %r4, .LBB7_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB7_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB7_1 +; CHECK-NEXT: .LBB7_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i64, ptr %src, i64 -65537 %res = atomicrmw min ptr %ptr, i64 %b seq_cst ret i64 %res @@ -120,10 +207,24 @@ define i64 @f8(i64 %dummy, ptr %src, i64 %b) { ; Check that indexed addresses are not allowed. define i64 @f9(i64 %dummy, i64 %base, i64 %index, i64 %b) { ; CHECK-LABEL: f9: -; CHECK: agr %r3, %r4 -; CHECK: lg %r2, 0(%r3) -; CHECK: csg %r2, {{%r[0-9]+}}, 0(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, 0(%r4,%r3) +; CHECK-NEXT: agr %r3, %r4 +; CHECK-NEXT: j .LBB8_2 +; CHECK-NEXT: .LBB8_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB8_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB8_4 +; CHECK-NEXT: .LBB8_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgrjle %r2, %r5, .LBB8_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB8_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r5 +; CHECK-NEXT: j .LBB8_1 +; CHECK-NEXT: .LBB8_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %add = add i64 %base, %index %ptr = inttoptr i64 %add to ptr %res = atomicrmw min ptr %ptr, i64 %b seq_cst @@ -133,17 +234,23 @@ define i64 @f9(i64 %dummy, i64 %base, i64 %index, i64 %b) { ; Check that constants are handled. define i64 @f10(i64 %dummy, ptr %ptr) { ; CHECK-LABEL: f10: -; CHECK-DAG: lghi [[LIMIT:%r[0-9]+]], 42 -; CHECK-DAG: lg %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: csg %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP:\.[^:]*]]: -; CHECK: lgr [[NEW:%r[0-9]+]], %r2 -; CHECK: cgrjle %r2, [[LIMIT]], [[KEEP:\..*]] -; CHECK: lghi [[NEW]], 42 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, 0(%r3) +; CHECK-NEXT: j .LBB9_2 +; CHECK-NEXT: .LBB9_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB9_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB9_4 +; CHECK-NEXT: .LBB9_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgijl %r2, 43, .LBB9_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB9_2 Depth=1 +; CHECK-NEXT: lghi %r0, 42 +; CHECK-NEXT: j .LBB9_1 +; CHECK-NEXT: .LBB9_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw min ptr %ptr, i64 42 seq_cst ret i64 %res } diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-nand-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-nand-03.ll index 323eafb3e5b3afcf10a8e592ff94679e2d2c6211..8e9870f13013f35ffbaad98f5dcc723be4e5e5cb 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-nand-03.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-nand-03.ll @@ -17,14 +17,14 @@ define i32 @f1(i32 %dummy, ptr %src, i32 %b) { ret i32 %res } -; Check NANDs of 1. +; Check NANDs with different constant operands. define i32 @f2(i32 %dummy, ptr %src) { ; CHECK-LABEL: f2: ; CHECK: l %r2, 0(%r3) ; CHECK: [[LABEL:\.[^ ]*]]: ; CHECK: lr %r0, %r2 -; CHECK: nilf %r0, 1 ; CHECK: xilf %r0, 4294967295 +; CHECK: oilf %r0, 4294967294 ; CHECK: cs %r2, %r0, 0(%r3) ; CHECK: jl [[LABEL]] ; CHECK: br %r14 @@ -32,61 +32,55 @@ define i32 @f2(i32 %dummy, ptr %src) { ret i32 %res } -; Check NANDs of the low end of the NILH range. define i32 @f3(i32 %dummy, ptr %src) { ; CHECK-LABEL: f3: -; CHECK: nilh %r0, 0 ; CHECK: xilf %r0, 4294967295 +; CHECK: oilh %r0, 65535 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i32 65535 seq_cst ret i32 %res } -; Check the next value up, which must use NILF. define i32 @f4(i32 %dummy, ptr %src) { ; CHECK-LABEL: f4: -; CHECK: nilf %r0, 65536 ; CHECK: xilf %r0, 4294967295 +; CHECK: oilf %r0, 4294901759 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i32 65536 seq_cst ret i32 %res } -; Check the largest useful NILL value. define i32 @f5(i32 %dummy, ptr %src) { ; CHECK-LABEL: f5: -; CHECK: nill %r0, 65534 ; CHECK: xilf %r0, 4294967295 +; CHECK: oill %r0, 1 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i32 -2 seq_cst ret i32 %res } -; Check the low end of the NILL range. define i32 @f6(i32 %dummy, ptr %src) { ; CHECK-LABEL: f6: -; CHECK: nill %r0, 0 ; CHECK: xilf %r0, 4294967295 +; CHECK: oill %r0, 65535 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i32 -65536 seq_cst ret i32 %res } -; Check the largest useful NILH value, which is one less than the above. define i32 @f7(i32 %dummy, ptr %src) { ; CHECK-LABEL: f7: -; CHECK: nilh %r0, 65534 ; CHECK: xilf %r0, 4294967295 +; CHECK: oilh %r0, 1 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i32 -65537 seq_cst ret i32 %res } -; Check the highest useful NILF value, which is one less than the above. define i32 @f8(i32 %dummy, ptr %src) { ; CHECK-LABEL: f8: -; CHECK: nilf %r0, 4294901758 ; CHECK: xilf %r0, 4294967295 +; CHECK: oilf %r0, 65537 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i32 -65538 seq_cst ret i32 %res diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-nand-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-nand-04.ll index b37030255d7cc1de8c6aec04f23a95154dcb77d1..3ff259d5576c8e8cb8afd7e79f2b97fc13c92e68 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-nand-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-nand-04.ll @@ -18,40 +18,38 @@ define i64 @f1(i64 %dummy, ptr %src, i64 %b) { ret i64 %res } -; Check NANDs of 1, which are done using a register. (We could use RISBG -; instead, but that isn't implemented yet.) +; Check NANDs of 1, which are done using a register. define i64 @f2(i64 %dummy, ptr %src) { ; CHECK-LABEL: f2: -; CHECK: ngr +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihf %r0, 4294967295 +; CHECK: oilf %r0, 4294967294 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 1 seq_cst ret i64 %res } -; Check the equivalent of NIHF with 1, which can use RISBG instead. define i64 @f3(i64 %dummy, ptr %src) { ; CHECK-LABEL: f3: ; CHECK: lg %r2, 0(%r3) ; CHECK: [[LABEL:\.[^:]*]]: -; CHECK: risbg %r0, %r2, 31, 191, 0 -; CHECK: lcgr %r0, %r0 +; CHECK: lcgr %r0, %r2 ; CHECK: aghi %r0, -1 -; CHECK: csg %r2, %r0, 0(%r3) +; CHECK: oihf %r0, 4294967294 ; CHECK: jl [[LABEL]] ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 8589934591 seq_cst ret i64 %res } -; Check the lowest NIHF value outside the range of RISBG. define i64 @f4(i64 %dummy, ptr %src) { ; CHECK-LABEL: f4: ; CHECK: lg %r2, 0(%r3) ; CHECK: [[LABEL:\.[^:]*]]: -; CHECK: lgr %r0, %r2 -; CHECK: nihf %r0, 2 -; CHECK: lcgr %r0, %r0 +; CHECK: lcgr %r0, %r2 ; CHECK: aghi %r0, -1 +; CHECK: oihf %r0, 4294967293 ; CHECK: csg %r2, %r0, 0(%r3) ; CHECK: jl [[LABEL]] ; CHECK: br %r14 @@ -59,118 +57,133 @@ define i64 @f4(i64 %dummy, ptr %src) { ret i64 %res } -; Check the next value up, which must use a register. define i64 @f5(i64 %dummy, ptr %src) { ; CHECK-LABEL: f5: -; CHECK: ngr +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihf %r0, 4294967292 +; CHECK: oilf %r0, 4294967295 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 12884901888 seq_cst ret i64 %res } -; Check the lowest NIHH value outside the range of RISBG. define i64 @f6(i64 %dummy, ptr %src) { ; CHECK-LABEL: f6: -; CHECK: nihh {{%r[0-5]}}, 2 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihh %r0, 65533 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 844424930131967 seq_cst ret i64 %res } -; Check the next value up, which must use a register. define i64 @f7(i64 %dummy, ptr %src) { ; CHECK-LABEL: f7: -; CHECK: ngr +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihf %r0, 4294901759 +; CHECK: oilf %r0, 4294967295 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 281474976710656 seq_cst ret i64 %res } -; Check the highest NILL value outside the range of RISBG. define i64 @f8(i64 %dummy, ptr %src) { ; CHECK-LABEL: f8: -; CHECK: nill {{%r[0-5]}}, 65530 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oill %r0, 5 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -6 seq_cst ret i64 %res } -; Check the lowest NILL value outside the range of RISBG. define i64 @f9(i64 %dummy, ptr %src) { ; CHECK-LABEL: f9: -; CHECK: nill {{%r[0-5]}}, 2 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oill %r0, 65533 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -65534 seq_cst ret i64 %res } -; Check the highest useful NILF value. define i64 @f10(i64 %dummy, ptr %src) { ; CHECK-LABEL: f10: -; CHECK: nilf {{%r[0-5]}}, 4294901758 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oilf %r0, 65537 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -65538 seq_cst ret i64 %res } -; Check the highest NILH value outside the range of RISBG. define i64 @f11(i64 %dummy, ptr %src) { ; CHECK-LABEL: f11: -; CHECK: nilh {{%r[0-5]}}, 65530 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oilh %r0, 5 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -327681 seq_cst ret i64 %res } -; Check the lowest NILH value outside the range of RISBG. define i64 @f12(i64 %dummy, ptr %src) { ; CHECK-LABEL: f12: -; CHECK: nilh {{%r[0-5]}}, 2 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oilh %r0, 65533 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -4294770689 seq_cst ret i64 %res } -; Check the lowest NILF value outside the range of RISBG. define i64 @f13(i64 %dummy, ptr %src) { ; CHECK-LABEL: f13: -; CHECK: nilf {{%r[0-5]}}, 2 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oilf %r0, 4294967293 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -4294967294 seq_cst ret i64 %res } -; Check the highest NIHL value outside the range of RISBG. define i64 @f14(i64 %dummy, ptr %src) { ; CHECK-LABEL: f14: -; CHECK: nihl {{%r[0-5]}}, 65530 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihl %r0, 5 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -21474836481 seq_cst ret i64 %res } -; Check the lowest NIHL value outside the range of RISBG. define i64 @f15(i64 %dummy, ptr %src) { ; CHECK-LABEL: f15: -; CHECK: nihl {{%r[0-5]}}, 2 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihl %r0, 65533 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -281462091808769 seq_cst ret i64 %res } -; Check the highest NIHH value outside the range of RISBG. define i64 @f16(i64 %dummy, ptr %src) { ; CHECK-LABEL: f16: -; CHECK: nihh {{%r[0-5]}}, 65530 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihh %r0, 5 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -1407374883553281 seq_cst ret i64 %res } -; Check the highest useful NIHF value. define i64 @f17(i64 %dummy, ptr %src) { ; CHECK-LABEL: f17: -; CHECK: nihf {{%r[0-5]}}, 4294901758 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihf %r0, 65537 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -281479271677953 seq_cst ret i64 %res diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-or-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-or-04.ll index a1322df49dbf8752e7a682d13e4edef465ce2c03..e29097b10e169187cb5652ac0483434e54fd06c9 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-or-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-or-04.ll @@ -93,11 +93,11 @@ define i64 @f9(i64 %dummy, ptr %src) { ret i64 %res } -; Check the next value up, which must use a register. (We could use -; combinations of OIH* and OIL* instead, but that isn't implemented.) +; Check the next value up, which must use a register. define i64 @f10(i64 %dummy, ptr %src) { ; CHECK-LABEL: f10: -; CHECK: ogr +; CHECK: oihl %r0, 1 +; CHECK: oill %r0, 1 ; CHECK: br %r14 %res = atomicrmw or ptr %src, i64 4294967297 seq_cst ret i64 %res @@ -139,10 +139,11 @@ define i64 @f14(i64 %dummy, ptr %src) { ret i64 %res } -; Check the next value up, which must use a register. +; Check the next value up. define i64 @f15(i64 %dummy, ptr %src) { ; CHECK-LABEL: f15: -; CHECK: ogr +; CHECK: oihh %r0, 65535 +; CHECK: oill %r0, 1 ; CHECK: br %r14 %res = atomicrmw or ptr %src, i64 18446462598732840961 seq_cst ret i64 %res diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-sub-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-sub-04.ll index 5d23d4e9ca1552adb5bf9a4a9d23a77aca8bf1eb..d18c72f3b41e74e82cba8d06e1ea72a90bf87884 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-sub-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-sub-04.ll @@ -16,13 +16,12 @@ define i64 @f1(i64 %dummy, ptr %src, i64 %b) { ret i64 %res } -; Check subtraction of 1, which can use AGHI. +; Check subtraction of 1. define i64 @f2(i64 %dummy, ptr %src) { ; CHECK-LABEL: f2: ; CHECK: lg %r2, 0(%r3) ; CHECK: [[LABEL:\.[^:]*]]: -; CHECK: lgr %r0, %r2 -; CHECK: aghi %r0, -1 +; CHECK: lay %r0, -1(%r2) ; CHECK: csg %r2, %r0, 0(%r3) ; CHECK: jl [[LABEL]] ; CHECK: br %r14 @@ -30,82 +29,64 @@ define i64 @f2(i64 %dummy, ptr %src) { ret i64 %res } -; Check the low end of the AGHI range. +; Check use of LAY. define i64 @f3(i64 %dummy, ptr %src) { ; CHECK-LABEL: f3: -; CHECK: aghi %r0, -32768 +; CHECK: lay %r0, -32768(%r2) ; CHECK: br %r14 %res = atomicrmw sub ptr %src, i64 32768 seq_cst ret i64 %res } -; Check the next value up, which must use AGFI. +; Check the low end of the AGFI range. define i64 @f4(i64 %dummy, ptr %src) { ; CHECK-LABEL: f4: -; CHECK: agfi %r0, -32769 +; CHECK: agfi %r0, -2147483648 ; CHECK: br %r14 - %res = atomicrmw sub ptr %src, i64 32769 seq_cst + %res = atomicrmw sub ptr %src, i64 2147483648 seq_cst ret i64 %res } -; Check the low end of the AGFI range. +; Check the next value up, which uses an SLGFI. define i64 @f5(i64 %dummy, ptr %src) { ; CHECK-LABEL: f5: -; CHECK: agfi %r0, -2147483648 +; CHECK: slgfi ; CHECK: br %r14 - %res = atomicrmw sub ptr %src, i64 2147483648 seq_cst + %res = atomicrmw sub ptr %src, i64 2147483649 seq_cst ret i64 %res } -; Check the next value up, which must use a register operation. +; Check subtraction of -1, which can use LA. define i64 @f6(i64 %dummy, ptr %src) { ; CHECK-LABEL: f6: -; CHECK: sgr +; CHECK: la %r0, 1(%r2) ; CHECK: br %r14 - %res = atomicrmw sub ptr %src, i64 2147483649 seq_cst + %res = atomicrmw sub ptr %src, i64 -1 seq_cst ret i64 %res } -; Check subtraction of -1, which can use AGHI. +; Check use of LAY. define i64 @f7(i64 %dummy, ptr %src) { ; CHECK-LABEL: f7: -; CHECK: aghi %r0, 1 -; CHECK: br %r14 - %res = atomicrmw sub ptr %src, i64 -1 seq_cst - ret i64 %res -} - -; Check the high end of the AGHI range. -define i64 @f8(i64 %dummy, ptr %src) { -; CHECK-LABEL: f8: -; CHECK: aghi %r0, 32767 +; CHECK: lay %r0, 32767(%r2) ; CHECK: br %r14 %res = atomicrmw sub ptr %src, i64 -32767 seq_cst ret i64 %res } -; Check the next value down, which must use AGFI instead. -define i64 @f9(i64 %dummy, ptr %src) { -; CHECK-LABEL: f9: -; CHECK: agfi %r0, 32768 -; CHECK: br %r14 - %res = atomicrmw sub ptr %src, i64 -32768 seq_cst - ret i64 %res -} - ; Check the high end of the AGFI range. -define i64 @f10(i64 %dummy, ptr %src) { -; CHECK-LABEL: f10: +define i64 @f8(i64 %dummy, ptr %src) { +; CHECK-LABEL: f8: ; CHECK: agfi %r0, 2147483647 ; CHECK: br %r14 %res = atomicrmw sub ptr %src, i64 -2147483647 seq_cst ret i64 %res } -; Check the next value down, which must use a register operation. -define i64 @f11(i64 %dummy, ptr %src) { -; CHECK-LABEL: f11: -; CHECK: sgr +; Check the next value down, which must use an ALGFI. +define i64 @f9(i64 %dummy, ptr %src) { +; CHECK-LABEL: f9: +; CHECK: algfi ; CHECK: br %r14 %res = atomicrmw sub ptr %src, i64 -2147483648 seq_cst ret i64 %res diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-03.ll index cf366b9f2e29ad7b34f2caa4b98ef00f79d12dce..d991997245089086b516701bfbe65890709d2ff3 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-03.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-03.ll @@ -96,8 +96,8 @@ define i32 @f8(i32 %dummy, ptr %src, i32 %b) { ; Check that indexed addresses are not allowed. define i32 @f9(i32 %dummy, i64 %base, i64 %index, i32 %b) { ; CHECK-LABEL: f9: +; CHECK: l %r2, 0(%r4,%r3) ; CHECK: agr %r3, %r4 -; CHECK: l %r2, 0(%r3) ; CHECK: cs %r2, {{%r[0-9]+}}, 0(%r3) ; CHECK: br %r14 %add = add i64 %base, %index diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-04.ll index 9a493cb7fd8c701eb603acff78d64a6739a350de..4797c5cf06798b6af75cbd356a724eb81100e047 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-04.ll @@ -63,8 +63,8 @@ define i64 @f5(i64 %dummy, ptr %src, i64 %b) { ; Check that indexed addresses are not allowed. define i64 @f6(i64 %dummy, i64 %base, i64 %index, i64 %b) { ; CHECK-LABEL: f6: +; CHECK: lg %r2, 0(%r4,%r3) ; CHECK: agr %r3, %r4 -; CHECK: lg %r2, 0(%r3) ; CHECK: csg %r2, {{%r[0-9]+}}, 0(%r3) ; CHECK: br %r14 %add = add i64 %base, %index diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xor-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xor-04.ll index 6cf1b80b8d0cd1803af317d0dbda0c0139ad6ba0..ee5fc1cf415dc9e492daef75beb1b1b8ee047d70 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-xor-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xor-04.ll @@ -48,11 +48,11 @@ define i64 @f4(i64 %dummy, ptr %src) { ret i64 %res } -; Check the next value up, which must use a register. (We could use -; combinations of XIH* and XIL* instead, but that isn't implemented.) +; Check the next value up. define i64 @f5(i64 %dummy, ptr %src) { ; CHECK-LABEL: f5: -; CHECK: xgr +; CHECK: xihf %r0, 1 +; CHECK: xilf %r0, 1 ; CHECK: br %r14 %res = atomicrmw xor ptr %src, i64 4294967297 seq_cst ret i64 %res @@ -70,7 +70,8 @@ define i64 @f6(i64 %dummy, ptr %src) { ; Check the next value up, which must use a register. define i64 @f7(i64 %dummy, ptr %src) { ; CHECK-LABEL: f7: -; CHECK: xgr +; CHECK: xihf %r0, 4294967295 +; CHECK: xilf %r0, 1 ; CHECK: br %r14 %res = atomicrmw xor ptr %src, i64 -4294967295 seq_cst ret i64 %res diff --git a/llvm/test/CodeGen/SystemZ/xor-04.ll b/llvm/test/CodeGen/SystemZ/xor-04.ll index 44f0a4cc39d0c3e1309375ff66066c35b0c9f3b3..ce5b76507b0dc2fe5a583350bba71885a32cea7e 100644 --- a/llvm/test/CodeGen/SystemZ/xor-04.ll +++ b/llvm/test/CodeGen/SystemZ/xor-04.ll @@ -61,8 +61,8 @@ define i64 @f6(i64 %a) { ; Check full bitwise negation define i64 @f7(i64 %a) { ; CHECK-LABEL: f7: -; CHECK: xihf %r2, 4294967295 -; CHECK: xilf %r2, 4294967295 +; CHECK: lcgr %r2, %r2 +; CHECK: aghi %r2, -1 ; CHECK: br %r14 %xor = xor i64 %a, -1 ret i64 %xor diff --git a/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll b/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll index cc7b5a7f3f819fdaf80b756fa1ea4fe4cd2d3bba..7087041e8dace67d484e1037d2c9b664dd8a2434 100644 --- a/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll +++ b/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll @@ -385,19 +385,17 @@ define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) { ; CHECK-NEXT: @ implicit-def: $r11 ; CHECK-NEXT: mov.w r9, #12 ; CHECK-NEXT: str r4, [sp, #12] @ 4-byte Spill -; CHECK-NEXT: add.w r0, r0, r2, lsr #1 ; CHECK-NEXT: add.w r1, r1, r2, lsr #1 -; CHECK-NEXT: movw r2, #65532 -; CHECK-NEXT: vdup.32 q6, r0 -; CHECK-NEXT: movt r2, #32767 -; CHECK-NEXT: and.w r3, r1, r2 +; CHECK-NEXT: add.w r0, r0, r2, lsr #1 +; CHECK-NEXT: bic r3, r1, #3 ; CHECK-NEXT: adr r1, .LCPI1_0 -; CHECK-NEXT: vdup.32 q7, r0 ; CHECK-NEXT: vldrw.u32 q0, [r1] ; CHECK-NEXT: adr r1, .LCPI1_1 ; CHECK-NEXT: vldrw.u32 q5, [r1] -; CHECK-NEXT: strd r3, r7, [sp, #4] @ 8-byte Folded Spill +; CHECK-NEXT: vdup.32 q6, r0 ; CHECK-NEXT: vadd.i32 q4, q0, r7 +; CHECK-NEXT: vdup.32 q7, r0 +; CHECK-NEXT: strd r3, r7, [sp, #4] @ 8-byte Folded Spill ; CHECK-NEXT: b .LBB1_6 ; CHECK-NEXT: .LBB1_2: @ %for.body6.preheader ; CHECK-NEXT: @ in Loop: Header=BB1_6 Depth=1 diff --git a/llvm/test/CodeGen/WebAssembly/wasm-eh-prepare.ll b/llvm/test/CodeGen/WebAssembly/wasm-eh-prepare.ll index e3de251017384ab9256304247947c584dfc0f787..bd577e387c72b7fa8def7e678b8ad554067e31a9 100644 --- a/llvm/test/CodeGen/WebAssembly/wasm-eh-prepare.ll +++ b/llvm/test/CodeGen/WebAssembly/wasm-eh-prepare.ll @@ -1,5 +1,5 @@ -; RUN: opt < %s -winehprepare -demote-catchswitch-only -wasm-eh-prepare -S | FileCheck %s -; RUN: opt < %s -winehprepare -demote-catchswitch-only -wasm-eh-prepare -S --mattr=+atomics,+bulk-memory | FileCheck %s +; RUN: opt < %s -win-eh-prepare -demote-catchswitch-only -wasm-eh-prepare -S | FileCheck %s +; RUN: opt < %s -win-eh-prepare -demote-catchswitch-only -wasm-eh-prepare -S --mattr=+atomics,+bulk-memory | FileCheck %s ; RUN: opt < %s -passes='win-eh-prepare,wasm-eh-prepare' -S | FileCheck %s ; RUN: opt < %s -passes='win-eh-prepare,wasm-eh-prepare' -S --mattr=+atomics,+bulk-memory | FileCheck %s diff --git a/llvm/test/CodeGen/WinEH/wineh-asm.ll b/llvm/test/CodeGen/WinEH/wineh-asm.ll index 00bb6ccb0a263e0d80d55c44e267b4e85dec99a1..7aa29bb2be45756643da94f71bef750892fbab65 100644 --- a/llvm/test/CodeGen/WinEH/wineh-asm.ll +++ b/llvm/test/CodeGen/WinEH/wineh-asm.ll @@ -1,4 +1,4 @@ -; RUN: opt -winehprepare < %s +; RUN: opt -win-eh-prepare < %s ; RUN: opt -passes=win-eh-prepare < %s target triple = "x86_64-pc-windows-msvc" diff --git a/llvm/test/CodeGen/WinEH/wineh-cloning.ll b/llvm/test/CodeGen/WinEH/wineh-cloning.ll index 9a006326fed6354f2a145dda9e92ffc55eda3d71..5df2eb26ead9648b8b9fa7e50e7058511ab01a42 100644 --- a/llvm/test/CodeGen/WinEH/wineh-cloning.ll +++ b/llvm/test/CodeGen/WinEH/wineh-cloning.ll @@ -1,4 +1,4 @@ -; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -winehprepare < %s | FileCheck %s +; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -win-eh-prepare < %s | FileCheck %s ; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -passes=win-eh-prepare < %s | FileCheck %s declare i32 @__CxxFrameHandler3(...) diff --git a/llvm/test/CodeGen/WinEH/wineh-demotion.ll b/llvm/test/CodeGen/WinEH/wineh-demotion.ll index 326bb2d5f874f456234062cb2ccece299cd1a24b..36a21e29f9c389932dcbaf94abf6ccbe3927a3ae 100644 --- a/llvm/test/CodeGen/WinEH/wineh-demotion.ll +++ b/llvm/test/CodeGen/WinEH/wineh-demotion.ll @@ -1,4 +1,4 @@ -; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -winehprepare < %s | FileCheck %s +; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -win-eh-prepare < %s | FileCheck %s ; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -passes=win-eh-prepare < %s | FileCheck %s declare i32 @__CxxFrameHandler3(...) diff --git a/llvm/test/CodeGen/WinEH/wineh-no-demotion.ll b/llvm/test/CodeGen/WinEH/wineh-no-demotion.ll index eebaf6d1c5be425cfcbead887f500889caa62f10..d4667db97cc141740c684957eb11c80e5fbe4751 100644 --- a/llvm/test/CodeGen/WinEH/wineh-no-demotion.ll +++ b/llvm/test/CodeGen/WinEH/wineh-no-demotion.ll @@ -1,4 +1,4 @@ -; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -winehprepare -disable-demotion -disable-cleanups < %s | FileCheck %s +; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -win-eh-prepare -disable-demotion -disable-cleanups < %s | FileCheck %s ; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -passes=win-eh-prepare -disable-demotion -disable-cleanups < %s | FileCheck %s declare i32 @__CxxFrameHandler3(...) diff --git a/llvm/test/CodeGen/X86/2011-10-19-widen_vselect.ll b/llvm/test/CodeGen/X86/2011-10-19-widen_vselect.ll index e7f62b9dfc221962560df5253e2d3d3870612c40..171e16e35fc2f0bfd8708970a6520d359f042187 100644 --- a/llvm/test/CodeGen/X86/2011-10-19-widen_vselect.ll +++ b/llvm/test/CodeGen/X86/2011-10-19-widen_vselect.ll @@ -50,13 +50,12 @@ define void @zero_test() { ; X86-LABEL: zero_test: ; X86: # %bb.0: # %entry ; X86-NEXT: xorps %xmm0, %xmm0 -; X86-NEXT: movlps %xmm0, (%eax) +; X86-NEXT: movsd %xmm0, (%eax) ; X86-NEXT: retl ; ; X64-LABEL: zero_test: ; X64: # %bb.0: # %entry -; X64-NEXT: xorps %xmm0, %xmm0 -; X64-NEXT: movlps %xmm0, (%rax) +; X64-NEXT: movq $0, (%rax) ; X64-NEXT: retq entry: %0 = select <2 x i1> undef, <2 x float> undef, <2 x float> zeroinitializer diff --git a/llvm/test/CodeGen/X86/2012-07-10-extload64.ll b/llvm/test/CodeGen/X86/2012-07-10-extload64.ll index b6ec3b34eb1072d031a1a45adfe641f4db0ce823..6db17251cd599f5a66440a5ba7dc1f197978dffb 100644 --- a/llvm/test/CodeGen/X86/2012-07-10-extload64.ll +++ b/llvm/test/CodeGen/X86/2012-07-10-extload64.ll @@ -30,7 +30,7 @@ define void @store_64(ptr %ptr) { ; X86: # %bb.0: # %BB ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: xorps %xmm0, %xmm0 -; X86-NEXT: movlps %xmm0, (%eax) +; X86-NEXT: movsd %xmm0, (%eax) ; X86-NEXT: retl ; ; X64-LABEL: store_64: diff --git a/llvm/test/CodeGen/X86/atomic-nocx16.ll b/llvm/test/CodeGen/X86/atomic-nocx16.ll new file mode 100644 index 0000000000000000000000000000000000000000..5677541242a2491148f623d1d8874e9fa39da803 --- /dev/null +++ b/llvm/test/CodeGen/X86/atomic-nocx16.ll @@ -0,0 +1,49 @@ +; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=corei7 -mattr=-cx16 | FileCheck %s +; RUN: llc < %s -mtriple=i386-linux-gnu -verify-machineinstrs -mattr=cx16 | FileCheck -check-prefix=CHECK %s + +;; Verify that 128-bit atomics emit a libcall without cx16 +;; available. +;; +;; We test 32-bit mode with -mattr=cx16, because it should have no +;; effect for 32-bit mode. + +; CHECK-LABEL: test: +define void @test(ptr %a) nounwind { +entry: +; CHECK: __sync_val_compare_and_swap_16 + %0 = cmpxchg ptr %a, i128 1, i128 1 seq_cst seq_cst +; CHECK: __sync_lock_test_and_set_16 + %1 = atomicrmw xchg ptr %a, i128 1 seq_cst +; CHECK: __sync_fetch_and_add_16 + %2 = atomicrmw add ptr %a, i128 1 seq_cst +; CHECK: __sync_fetch_and_sub_16 + %3 = atomicrmw sub ptr %a, i128 1 seq_cst +; CHECK: __sync_fetch_and_and_16 + %4 = atomicrmw and ptr %a, i128 1 seq_cst +; CHECK: __sync_fetch_and_nand_16 + %5 = atomicrmw nand ptr %a, i128 1 seq_cst +; CHECK: __sync_fetch_and_or_16 + %6 = atomicrmw or ptr %a, i128 1 seq_cst +; CHECK: __sync_fetch_and_xor_16 + %7 = atomicrmw xor ptr %a, i128 1 seq_cst +; CHECK: __sync_val_compare_and_swap_16 + %8 = load atomic i128, ptr %a seq_cst, align 16 +; CHECK: __sync_lock_test_and_set_16 + store atomic i128 %8, ptr %a seq_cst, align 16 + ret void +} + +; CHECK-LABEL: test_fp: +define void @test_fp(fp128* %a) nounwind { +entry: +; CHECK: __sync_lock_test_and_set_16 + %0 = atomicrmw xchg fp128* %a, fp128 0xL00000000000000004000900000000000 seq_cst +; Currently fails to compile: +; %1 = atomicrmw fadd fp128* %a, fp128 0xL00000000000000004000900000000000 seq_cst +; %2 = atomicrmw fsub fp128* %a, fp128 0xL00000000000000004000900000000000 seq_cst +; CHECK: __sync_val_compare_and_swap_16 + %1 = load atomic fp128, fp128* %a seq_cst, align 16 +; CHECK: __sync_lock_test_and_set_16 + store atomic fp128 %1, fp128* %a seq_cst, align 16 + ret void +} diff --git a/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll b/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll index 6c4d112330f2e95cab571c64897e957f9c938e4c..9555c45086d6feb6b9683329f19a2ca5d1f22b64 100644 --- a/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll +++ b/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll @@ -1,35 +1,139 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=-sse | FileCheck %s --check-prefix=X64-NOSSE -; RUN: llc < %s -mtriple=x86_64-linux-generic -verify-machineinstrs | FileCheck %s --check-prefix=X64-SSE +; RUN: llc < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=sse2,cx16 | FileCheck %s --check-prefixes=X64-SSE +; RUN: llc < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=avx,cx16 | FileCheck %s --check-prefixes=X64-AVX +; RUN: llc < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=avx512f,cx16 | FileCheck %s --check-prefixes=X64-AVX -; Note: This test is testing that the lowering for atomics matches what we -; currently emit for non-atomics + the atomic restriction. The presence of -; particular lowering detail in these tests should not be read as requiring -; that detail for correctness unless it's related to the atomicity itself. -; (Specifically, there were reviewer questions about the lowering for halfs -; and their calling convention which remain unresolved.) +; Codegen of fp128 without cx16 is tested in atomic-nocx16.ll define void @store_fp128(ptr %fptr, fp128 %v) { -; X64-NOSSE-LABEL: store_fp128: -; X64-NOSSE: # %bb.0: -; X64-NOSSE-NEXT: pushq %rax -; X64-NOSSE-NEXT: .cfi_def_cfa_offset 16 -; X64-NOSSE-NEXT: callq __sync_lock_test_and_set_16@PLT -; X64-NOSSE-NEXT: popq %rax -; X64-NOSSE-NEXT: .cfi_def_cfa_offset 8 -; X64-NOSSE-NEXT: retq -; ; X64-SSE-LABEL: store_fp128: ; X64-SSE: # %bb.0: -; X64-SSE-NEXT: subq $24, %rsp -; X64-SSE-NEXT: .cfi_def_cfa_offset 32 -; X64-SSE-NEXT: movaps %xmm0, (%rsp) -; X64-SSE-NEXT: movq (%rsp), %rsi -; X64-SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx -; X64-SSE-NEXT: callq __sync_lock_test_and_set_16@PLT -; X64-SSE-NEXT: addq $24, %rsp +; X64-SSE-NEXT: pushq %rbx +; X64-SSE-NEXT: .cfi_def_cfa_offset 16 +; X64-SSE-NEXT: .cfi_offset %rbx, -16 +; X64-SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp) +; X64-SSE-NEXT: movq -{{[0-9]+}}(%rsp), %rbx +; X64-SSE-NEXT: movq -{{[0-9]+}}(%rsp), %rcx +; X64-SSE-NEXT: movq (%rdi), %rax +; X64-SSE-NEXT: movq 8(%rdi), %rdx +; X64-SSE-NEXT: .p2align 4, 0x90 +; X64-SSE-NEXT: .LBB0_1: # %atomicrmw.start +; X64-SSE-NEXT: # =>This Inner Loop Header: Depth=1 +; X64-SSE-NEXT: lock cmpxchg16b (%rdi) +; X64-SSE-NEXT: jne .LBB0_1 +; X64-SSE-NEXT: # %bb.2: # %atomicrmw.end +; X64-SSE-NEXT: popq %rbx ; X64-SSE-NEXT: .cfi_def_cfa_offset 8 ; X64-SSE-NEXT: retq +; +; X64-AVX-LABEL: store_fp128: +; X64-AVX: # %bb.0: +; X64-AVX-NEXT: pushq %rbx +; X64-AVX-NEXT: .cfi_def_cfa_offset 16 +; X64-AVX-NEXT: .cfi_offset %rbx, -16 +; X64-AVX-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp) +; X64-AVX-NEXT: movq -{{[0-9]+}}(%rsp), %rbx +; X64-AVX-NEXT: movq -{{[0-9]+}}(%rsp), %rcx +; X64-AVX-NEXT: movq (%rdi), %rax +; X64-AVX-NEXT: movq 8(%rdi), %rdx +; X64-AVX-NEXT: .p2align 4, 0x90 +; X64-AVX-NEXT: .LBB0_1: # %atomicrmw.start +; X64-AVX-NEXT: # =>This Inner Loop Header: Depth=1 +; X64-AVX-NEXT: lock cmpxchg16b (%rdi) +; X64-AVX-NEXT: jne .LBB0_1 +; X64-AVX-NEXT: # %bb.2: # %atomicrmw.end +; X64-AVX-NEXT: popq %rbx +; X64-AVX-NEXT: .cfi_def_cfa_offset 8 +; X64-AVX-NEXT: retq store atomic fp128 %v, ptr %fptr unordered, align 16 ret void } + +define fp128 @load_fp128(ptr %fptr) { +; X64-SSE-LABEL: load_fp128: +; X64-SSE: # %bb.0: +; X64-SSE-NEXT: pushq %rbx +; X64-SSE-NEXT: .cfi_def_cfa_offset 16 +; X64-SSE-NEXT: .cfi_offset %rbx, -16 +; X64-SSE-NEXT: xorl %eax, %eax +; X64-SSE-NEXT: xorl %edx, %edx +; X64-SSE-NEXT: xorl %ecx, %ecx +; X64-SSE-NEXT: xorl %ebx, %ebx +; X64-SSE-NEXT: lock cmpxchg16b (%rdi) +; X64-SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp) +; X64-SSE-NEXT: movq %rax, -{{[0-9]+}}(%rsp) +; X64-SSE-NEXT: movaps -{{[0-9]+}}(%rsp), %xmm0 +; X64-SSE-NEXT: popq %rbx +; X64-SSE-NEXT: .cfi_def_cfa_offset 8 +; X64-SSE-NEXT: retq +; +; X64-AVX-LABEL: load_fp128: +; X64-AVX: # %bb.0: +; X64-AVX-NEXT: pushq %rbx +; X64-AVX-NEXT: .cfi_def_cfa_offset 16 +; X64-AVX-NEXT: .cfi_offset %rbx, -16 +; X64-AVX-NEXT: xorl %eax, %eax +; X64-AVX-NEXT: xorl %edx, %edx +; X64-AVX-NEXT: xorl %ecx, %ecx +; X64-AVX-NEXT: xorl %ebx, %ebx +; X64-AVX-NEXT: lock cmpxchg16b (%rdi) +; X64-AVX-NEXT: movq %rdx, -{{[0-9]+}}(%rsp) +; X64-AVX-NEXT: movq %rax, -{{[0-9]+}}(%rsp) +; X64-AVX-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0 +; X64-AVX-NEXT: popq %rbx +; X64-AVX-NEXT: .cfi_def_cfa_offset 8 +; X64-AVX-NEXT: retq + %v = load atomic fp128, ptr %fptr unordered, align 16 + ret fp128 %v +} + +define fp128 @exchange_fp128(ptr %fptr, fp128 %x) { +; X64-SSE-LABEL: exchange_fp128: +; X64-SSE: # %bb.0: +; X64-SSE-NEXT: pushq %rbx +; X64-SSE-NEXT: .cfi_def_cfa_offset 16 +; X64-SSE-NEXT: .cfi_offset %rbx, -16 +; X64-SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp) +; X64-SSE-NEXT: movq -{{[0-9]+}}(%rsp), %rbx +; X64-SSE-NEXT: movq -{{[0-9]+}}(%rsp), %rcx +; X64-SSE-NEXT: movq (%rdi), %rax +; X64-SSE-NEXT: movq 8(%rdi), %rdx +; X64-SSE-NEXT: .p2align 4, 0x90 +; X64-SSE-NEXT: .LBB2_1: # %atomicrmw.start +; X64-SSE-NEXT: # =>This Inner Loop Header: Depth=1 +; X64-SSE-NEXT: lock cmpxchg16b (%rdi) +; X64-SSE-NEXT: jne .LBB2_1 +; X64-SSE-NEXT: # %bb.2: # %atomicrmw.end +; X64-SSE-NEXT: movq %rax, -{{[0-9]+}}(%rsp) +; X64-SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp) +; X64-SSE-NEXT: movaps -{{[0-9]+}}(%rsp), %xmm0 +; X64-SSE-NEXT: popq %rbx +; X64-SSE-NEXT: .cfi_def_cfa_offset 8 +; X64-SSE-NEXT: retq +; +; X64-AVX-LABEL: exchange_fp128: +; X64-AVX: # %bb.0: +; X64-AVX-NEXT: pushq %rbx +; X64-AVX-NEXT: .cfi_def_cfa_offset 16 +; X64-AVX-NEXT: .cfi_offset %rbx, -16 +; X64-AVX-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp) +; X64-AVX-NEXT: movq -{{[0-9]+}}(%rsp), %rbx +; X64-AVX-NEXT: movq -{{[0-9]+}}(%rsp), %rcx +; X64-AVX-NEXT: movq (%rdi), %rax +; X64-AVX-NEXT: movq 8(%rdi), %rdx +; X64-AVX-NEXT: .p2align 4, 0x90 +; X64-AVX-NEXT: .LBB2_1: # %atomicrmw.start +; X64-AVX-NEXT: # =>This Inner Loop Header: Depth=1 +; X64-AVX-NEXT: lock cmpxchg16b (%rdi) +; X64-AVX-NEXT: jne .LBB2_1 +; X64-AVX-NEXT: # %bb.2: # %atomicrmw.end +; X64-AVX-NEXT: movq %rax, -{{[0-9]+}}(%rsp) +; X64-AVX-NEXT: movq %rdx, -{{[0-9]+}}(%rsp) +; X64-AVX-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0 +; X64-AVX-NEXT: popq %rbx +; X64-AVX-NEXT: .cfi_def_cfa_offset 8 +; X64-AVX-NEXT: retq + %v = atomicrmw xchg ptr %fptr, fp128 %x monotonic, align 16 + ret fp128 %v +} + diff --git a/llvm/test/CodeGen/X86/atomic-non-integer.ll b/llvm/test/CodeGen/X86/atomic-non-integer.ll index 7d2810e57a25b5eda792091d2f2eb0ab109346f4..9e6f584d83112f643039966a1da383f2daba93a5 100644 --- a/llvm/test/CodeGen/X86/atomic-non-integer.ll +++ b/llvm/test/CodeGen/X86/atomic-non-integer.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=i386-linux-generic -verify-machineinstrs -mattr=sse | FileCheck %s --check-prefixes=X86,X86-SSE,X86-SSE1 -; RUN: llc < %s -mtriple=i386-linux-generic -verify-machineinstrs -mattr=sse2 | FileCheck %s --check-prefixes=X86,X86-SSE,X86-SSE2 +; RUN: llc < %s -mtriple=i386-linux-generic -verify-machineinstrs -mattr=sse | FileCheck %s --check-prefixes=X86,X86-SSE1 +; RUN: llc < %s -mtriple=i386-linux-generic -verify-machineinstrs -mattr=sse2 | FileCheck %s --check-prefixes=X86,X86-SSE2 ; RUN: llc < %s -mtriple=i386-linux-generic -verify-machineinstrs -mattr=avx | FileCheck %s --check-prefixes=X86,X86-AVX ; RUN: llc < %s -mtriple=i386-linux-generic -verify-machineinstrs -mattr=avx512f | FileCheck %s --check-prefixes=X86,X86-AVX ; RUN: llc < %s -mtriple=i386-linux-generic -verify-machineinstrs | FileCheck %s --check-prefixes=X86,X86-NOSSE @@ -131,94 +131,6 @@ define void @store_double(ptr %fptr, double %v) { ret void } -define void @store_fp128(ptr %fptr, fp128 %v) { -; X86-SSE-LABEL: store_fp128: -; X86-SSE: # %bb.0: -; X86-SSE-NEXT: subl $36, %esp -; X86-SSE-NEXT: .cfi_adjust_cfa_offset 36 -; X86-SSE-NEXT: leal {{[0-9]+}}(%esp), %eax -; X86-SSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-SSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-SSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-SSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-SSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-SSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE-NEXT: pushl %eax -; X86-SSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE-NEXT: calll __sync_lock_test_and_set_16 -; X86-SSE-NEXT: .cfi_adjust_cfa_offset -4 -; X86-SSE-NEXT: addl $56, %esp -; X86-SSE-NEXT: .cfi_adjust_cfa_offset -56 -; X86-SSE-NEXT: retl -; -; X86-AVX-LABEL: store_fp128: -; X86-AVX: # %bb.0: -; X86-AVX-NEXT: subl $60, %esp -; X86-AVX-NEXT: .cfi_def_cfa_offset 64 -; X86-AVX-NEXT: vmovaps {{[0-9]+}}(%esp), %xmm0 -; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-AVX-NEXT: movl %eax, {{[0-9]+}}(%esp) -; X86-AVX-NEXT: vmovups %xmm0, {{[0-9]+}}(%esp) -; X86-AVX-NEXT: leal {{[0-9]+}}(%esp), %eax -; X86-AVX-NEXT: movl %eax, (%esp) -; X86-AVX-NEXT: calll __sync_lock_test_and_set_16 -; X86-AVX-NEXT: addl $56, %esp -; X86-AVX-NEXT: .cfi_def_cfa_offset 4 -; X86-AVX-NEXT: retl -; -; X86-NOSSE-LABEL: store_fp128: -; X86-NOSSE: # %bb.0: -; X86-NOSSE-NEXT: subl $36, %esp -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 36 -; X86-NOSSE-NEXT: leal {{[0-9]+}}(%esp), %eax -; X86-NOSSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl %eax -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: calll __sync_lock_test_and_set_16 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset -4 -; X86-NOSSE-NEXT: addl $56, %esp -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset -56 -; X86-NOSSE-NEXT: retl -; -; X64-SSE-LABEL: store_fp128: -; X64-SSE: # %bb.0: -; X64-SSE-NEXT: subq $24, %rsp -; X64-SSE-NEXT: .cfi_def_cfa_offset 32 -; X64-SSE-NEXT: movaps %xmm0, (%rsp) -; X64-SSE-NEXT: movq (%rsp), %rsi -; X64-SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx -; X64-SSE-NEXT: callq __sync_lock_test_and_set_16@PLT -; X64-SSE-NEXT: addq $24, %rsp -; X64-SSE-NEXT: .cfi_def_cfa_offset 8 -; X64-SSE-NEXT: retq -; -; X64-AVX-LABEL: store_fp128: -; X64-AVX: # %bb.0: -; X64-AVX-NEXT: subq $24, %rsp -; X64-AVX-NEXT: .cfi_def_cfa_offset 32 -; X64-AVX-NEXT: vmovaps %xmm0, (%rsp) -; X64-AVX-NEXT: movq (%rsp), %rsi -; X64-AVX-NEXT: movq {{[0-9]+}}(%rsp), %rdx -; X64-AVX-NEXT: callq __sync_lock_test_and_set_16@PLT -; X64-AVX-NEXT: addq $24, %rsp -; X64-AVX-NEXT: .cfi_def_cfa_offset 8 -; X64-AVX-NEXT: retq - store atomic fp128 %v, ptr %fptr unordered, align 16 - ret void -} define half @load_half(ptr %fptr) { ; X86-SSE1-LABEL: load_half: @@ -393,220 +305,273 @@ define double @load_double(ptr %fptr) { ret double %v } -define fp128 @load_fp128(ptr %fptr) { -; X86-SSE1-LABEL: load_fp128: +define half @exchange_half(ptr %fptr, half %x) { +; X86-SSE1-LABEL: exchange_half: +; X86-SSE1: # %bb.0: +; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-SSE1-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; X86-SSE1-NEXT: xchgw %ax, (%ecx) +; X86-SSE1-NEXT: retl +; +; X86-SSE2-LABEL: exchange_half: +; X86-SSE2: # %bb.0: +; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-SSE2-NEXT: movzwl {{[0-9]+}}(%esp), %ecx +; X86-SSE2-NEXT: xchgw %cx, (%eax) +; X86-SSE2-NEXT: pinsrw $0, %ecx, %xmm0 +; X86-SSE2-NEXT: retl +; +; X86-AVX-LABEL: exchange_half: +; X86-AVX: # %bb.0: +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-AVX-NEXT: movzwl {{[0-9]+}}(%esp), %ecx +; X86-AVX-NEXT: xchgw %cx, (%eax) +; X86-AVX-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm0 +; X86-AVX-NEXT: retl +; +; X86-NOSSE-LABEL: exchange_half: +; X86-NOSSE: # %bb.0: +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NOSSE-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; X86-NOSSE-NEXT: xchgw %ax, (%ecx) +; X86-NOSSE-NEXT: retl +; +; X64-SSE-LABEL: exchange_half: +; X64-SSE: # %bb.0: +; X64-SSE-NEXT: pextrw $0, %xmm0, %eax +; X64-SSE-NEXT: xchgw %ax, (%rdi) +; X64-SSE-NEXT: pinsrw $0, %eax, %xmm0 +; X64-SSE-NEXT: retq +; +; X64-AVX-LABEL: exchange_half: +; X64-AVX: # %bb.0: +; X64-AVX-NEXT: vpextrw $0, %xmm0, %eax +; X64-AVX-NEXT: xchgw %ax, (%rdi) +; X64-AVX-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0 +; X64-AVX-NEXT: retq + %v = atomicrmw xchg ptr %fptr, half %x monotonic, align 2 + ret half %v +} + +define float @exchange_float(ptr %fptr, float %x) { +; X86-SSE1-LABEL: exchange_float: ; X86-SSE1: # %bb.0: -; X86-SSE1-NEXT: pushl %edi +; X86-SSE1-NEXT: pushl %eax +; X86-SSE1-NEXT: .cfi_def_cfa_offset 8 +; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-SSE1-NEXT: xchgl %ecx, (%eax) +; X86-SSE1-NEXT: movl %ecx, (%esp) +; X86-SSE1-NEXT: flds (%esp) +; X86-SSE1-NEXT: popl %eax +; X86-SSE1-NEXT: .cfi_def_cfa_offset 4 +; X86-SSE1-NEXT: retl +; +; X86-SSE2-LABEL: exchange_float: +; X86-SSE2: # %bb.0: +; X86-SSE2-NEXT: pushl %eax +; X86-SSE2-NEXT: .cfi_def_cfa_offset 8 +; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-SSE2-NEXT: xchgl %ecx, (%eax) +; X86-SSE2-NEXT: movd %ecx, %xmm0 +; X86-SSE2-NEXT: movd %xmm0, (%esp) +; X86-SSE2-NEXT: flds (%esp) +; X86-SSE2-NEXT: popl %eax +; X86-SSE2-NEXT: .cfi_def_cfa_offset 4 +; X86-SSE2-NEXT: retl +; +; X86-AVX-LABEL: exchange_float: +; X86-AVX: # %bb.0: +; X86-AVX-NEXT: pushl %eax +; X86-AVX-NEXT: .cfi_def_cfa_offset 8 +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-AVX-NEXT: xchgl %ecx, (%eax) +; X86-AVX-NEXT: vmovd %ecx, %xmm0 +; X86-AVX-NEXT: vmovd %xmm0, (%esp) +; X86-AVX-NEXT: flds (%esp) +; X86-AVX-NEXT: popl %eax +; X86-AVX-NEXT: .cfi_def_cfa_offset 4 +; X86-AVX-NEXT: retl +; +; X86-NOSSE-LABEL: exchange_float: +; X86-NOSSE: # %bb.0: +; X86-NOSSE-NEXT: pushl %eax +; X86-NOSSE-NEXT: .cfi_def_cfa_offset 8 +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NOSSE-NEXT: xchgl %ecx, (%eax) +; X86-NOSSE-NEXT: movl %ecx, (%esp) +; X86-NOSSE-NEXT: flds (%esp) +; X86-NOSSE-NEXT: popl %eax +; X86-NOSSE-NEXT: .cfi_def_cfa_offset 4 +; X86-NOSSE-NEXT: retl +; +; X64-SSE-LABEL: exchange_float: +; X64-SSE: # %bb.0: +; X64-SSE-NEXT: movd %xmm0, %eax +; X64-SSE-NEXT: xchgl %eax, (%rdi) +; X64-SSE-NEXT: movd %eax, %xmm0 +; X64-SSE-NEXT: retq +; +; X64-AVX-LABEL: exchange_float: +; X64-AVX: # %bb.0: +; X64-AVX-NEXT: vmovd %xmm0, %eax +; X64-AVX-NEXT: xchgl %eax, (%rdi) +; X64-AVX-NEXT: vmovd %eax, %xmm0 +; X64-AVX-NEXT: retq + %v = atomicrmw xchg ptr %fptr, float %x monotonic, align 4 + ret float %v +} + +define double @exchange_double(ptr %fptr, double %x) { +; X86-SSE1-LABEL: exchange_double: +; X86-SSE1: # %bb.0: +; X86-SSE1-NEXT: pushl %ebx ; X86-SSE1-NEXT: .cfi_def_cfa_offset 8 ; X86-SSE1-NEXT: pushl %esi ; X86-SSE1-NEXT: .cfi_def_cfa_offset 12 -; X86-SSE1-NEXT: subl $20, %esp -; X86-SSE1-NEXT: .cfi_def_cfa_offset 32 +; X86-SSE1-NEXT: subl $12, %esp +; X86-SSE1-NEXT: .cfi_def_cfa_offset 24 ; X86-SSE1-NEXT: .cfi_offset %esi, -12 -; X86-SSE1-NEXT: .cfi_offset %edi, -8 +; X86-SSE1-NEXT: .cfi_offset %ebx, -8 ; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-SSE1-NEXT: subl $8, %esp -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 8 -; X86-SSE1-NEXT: leal {{[0-9]+}}(%esp), %eax -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl {{[0-9]+}}(%esp) -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl %eax -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: calll __sync_val_compare_and_swap_16 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset -4 -; X86-SSE1-NEXT: addl $44, %esp -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset -44 -; X86-SSE1-NEXT: movl (%esp), %eax +; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %ebx ; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %edi -; X86-SSE1-NEXT: movl %edi, 8(%esi) -; X86-SSE1-NEXT: movl %edx, 12(%esi) -; X86-SSE1-NEXT: movl %eax, (%esi) -; X86-SSE1-NEXT: movl %ecx, 4(%esi) -; X86-SSE1-NEXT: movl %esi, %eax -; X86-SSE1-NEXT: addl $20, %esp +; X86-SSE1-NEXT: movl (%esi), %eax +; X86-SSE1-NEXT: movl 4(%esi), %edx +; X86-SSE1-NEXT: .p2align 4, 0x90 +; X86-SSE1-NEXT: .LBB8_1: # %atomicrmw.start +; X86-SSE1-NEXT: # =>This Inner Loop Header: Depth=1 +; X86-SSE1-NEXT: lock cmpxchg8b (%esi) +; X86-SSE1-NEXT: jne .LBB8_1 +; X86-SSE1-NEXT: # %bb.2: # %atomicrmw.end +; X86-SSE1-NEXT: movl %eax, (%esp) +; X86-SSE1-NEXT: movl %edx, {{[0-9]+}}(%esp) +; X86-SSE1-NEXT: fldl (%esp) +; X86-SSE1-NEXT: addl $12, %esp ; X86-SSE1-NEXT: .cfi_def_cfa_offset 12 ; X86-SSE1-NEXT: popl %esi ; X86-SSE1-NEXT: .cfi_def_cfa_offset 8 -; X86-SSE1-NEXT: popl %edi +; X86-SSE1-NEXT: popl %ebx ; X86-SSE1-NEXT: .cfi_def_cfa_offset 4 -; X86-SSE1-NEXT: retl $4 +; X86-SSE1-NEXT: retl ; -; X86-SSE2-LABEL: load_fp128: +; X86-SSE2-LABEL: exchange_double: ; X86-SSE2: # %bb.0: -; X86-SSE2-NEXT: pushl %esi +; X86-SSE2-NEXT: pushl %ebx ; X86-SSE2-NEXT: .cfi_def_cfa_offset 8 -; X86-SSE2-NEXT: subl $24, %esp -; X86-SSE2-NEXT: .cfi_def_cfa_offset 32 -; X86-SSE2-NEXT: .cfi_offset %esi, -8 +; X86-SSE2-NEXT: pushl %esi +; X86-SSE2-NEXT: .cfi_def_cfa_offset 12 +; X86-SSE2-NEXT: subl $12, %esp +; X86-SSE2-NEXT: .cfi_def_cfa_offset 24 +; X86-SSE2-NEXT: .cfi_offset %esi, -12 +; X86-SSE2-NEXT: .cfi_offset %ebx, -8 ; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-SSE2-NEXT: subl $8, %esp -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 8 -; X86-SSE2-NEXT: leal {{[0-9]+}}(%esp), %eax -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl {{[0-9]+}}(%esp) -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl %eax -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: calll __sync_val_compare_and_swap_16 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset -4 -; X86-SSE2-NEXT: addl $44, %esp -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset -44 -; X86-SSE2-NEXT: movaps (%esp), %xmm0 -; X86-SSE2-NEXT: movaps %xmm0, (%esi) -; X86-SSE2-NEXT: movl %esi, %eax -; X86-SSE2-NEXT: addl $24, %esp -; X86-SSE2-NEXT: .cfi_def_cfa_offset 8 +; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebx +; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-SSE2-NEXT: movl (%esi), %eax +; X86-SSE2-NEXT: movl 4(%esi), %edx +; X86-SSE2-NEXT: .p2align 4, 0x90 +; X86-SSE2-NEXT: .LBB8_1: # %atomicrmw.start +; X86-SSE2-NEXT: # =>This Inner Loop Header: Depth=1 +; X86-SSE2-NEXT: lock cmpxchg8b (%esi) +; X86-SSE2-NEXT: jne .LBB8_1 +; X86-SSE2-NEXT: # %bb.2: # %atomicrmw.end +; X86-SSE2-NEXT: movd %eax, %xmm0 +; X86-SSE2-NEXT: movd %edx, %xmm1 +; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; X86-SSE2-NEXT: movq %xmm0, (%esp) +; X86-SSE2-NEXT: fldl (%esp) +; X86-SSE2-NEXT: addl $12, %esp +; X86-SSE2-NEXT: .cfi_def_cfa_offset 12 ; X86-SSE2-NEXT: popl %esi +; X86-SSE2-NEXT: .cfi_def_cfa_offset 8 +; X86-SSE2-NEXT: popl %ebx ; X86-SSE2-NEXT: .cfi_def_cfa_offset 4 -; X86-SSE2-NEXT: retl $4 +; X86-SSE2-NEXT: retl ; -; X86-AVX-LABEL: load_fp128: +; X86-AVX-LABEL: exchange_double: ; X86-AVX: # %bb.0: -; X86-AVX-NEXT: pushl %esi +; X86-AVX-NEXT: pushl %ebx ; X86-AVX-NEXT: .cfi_def_cfa_offset 8 -; X86-AVX-NEXT: subl $72, %esp -; X86-AVX-NEXT: .cfi_def_cfa_offset 80 -; X86-AVX-NEXT: .cfi_offset %esi, -8 +; X86-AVX-NEXT: pushl %esi +; X86-AVX-NEXT: .cfi_def_cfa_offset 12 +; X86-AVX-NEXT: subl $12, %esp +; X86-AVX-NEXT: .cfi_def_cfa_offset 24 +; X86-AVX-NEXT: .cfi_offset %esi, -12 +; X86-AVX-NEXT: .cfi_offset %ebx, -8 ; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0 -; X86-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp) -; X86-AVX-NEXT: movl %eax, {{[0-9]+}}(%esp) -; X86-AVX-NEXT: leal {{[0-9]+}}(%esp), %eax -; X86-AVX-NEXT: movl %eax, (%esp) -; X86-AVX-NEXT: vzeroupper -; X86-AVX-NEXT: calll __sync_val_compare_and_swap_16 -; X86-AVX-NEXT: subl $4, %esp -; X86-AVX-NEXT: vmovaps {{[0-9]+}}(%esp), %xmm0 -; X86-AVX-NEXT: vmovaps %xmm0, (%esi) -; X86-AVX-NEXT: movl %esi, %eax -; X86-AVX-NEXT: addl $72, %esp -; X86-AVX-NEXT: .cfi_def_cfa_offset 8 +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %ebx +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-AVX-NEXT: movl (%esi), %eax +; X86-AVX-NEXT: movl 4(%esi), %edx +; X86-AVX-NEXT: .p2align 4, 0x90 +; X86-AVX-NEXT: .LBB8_1: # %atomicrmw.start +; X86-AVX-NEXT: # =>This Inner Loop Header: Depth=1 +; X86-AVX-NEXT: lock cmpxchg8b (%esi) +; X86-AVX-NEXT: jne .LBB8_1 +; X86-AVX-NEXT: # %bb.2: # %atomicrmw.end +; X86-AVX-NEXT: vmovd %eax, %xmm0 +; X86-AVX-NEXT: vpinsrd $1, %edx, %xmm0, %xmm0 +; X86-AVX-NEXT: vmovq %xmm0, (%esp) +; X86-AVX-NEXT: fldl (%esp) +; X86-AVX-NEXT: addl $12, %esp +; X86-AVX-NEXT: .cfi_def_cfa_offset 12 ; X86-AVX-NEXT: popl %esi +; X86-AVX-NEXT: .cfi_def_cfa_offset 8 +; X86-AVX-NEXT: popl %ebx ; X86-AVX-NEXT: .cfi_def_cfa_offset 4 -; X86-AVX-NEXT: retl $4 +; X86-AVX-NEXT: retl ; -; X86-NOSSE-LABEL: load_fp128: +; X86-NOSSE-LABEL: exchange_double: ; X86-NOSSE: # %bb.0: -; X86-NOSSE-NEXT: pushl %edi +; X86-NOSSE-NEXT: pushl %ebx ; X86-NOSSE-NEXT: .cfi_def_cfa_offset 8 ; X86-NOSSE-NEXT: pushl %esi ; X86-NOSSE-NEXT: .cfi_def_cfa_offset 12 -; X86-NOSSE-NEXT: subl $20, %esp -; X86-NOSSE-NEXT: .cfi_def_cfa_offset 32 +; X86-NOSSE-NEXT: subl $12, %esp +; X86-NOSSE-NEXT: .cfi_def_cfa_offset 24 ; X86-NOSSE-NEXT: .cfi_offset %esi, -12 -; X86-NOSSE-NEXT: .cfi_offset %edi, -8 +; X86-NOSSE-NEXT: .cfi_offset %ebx, -8 ; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-NOSSE-NEXT: subl $8, %esp -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 8 -; X86-NOSSE-NEXT: leal {{[0-9]+}}(%esp), %eax -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl %eax -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: calll __sync_val_compare_and_swap_16 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset -4 -; X86-NOSSE-NEXT: addl $44, %esp -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset -44 -; X86-NOSSE-NEXT: movl (%esp), %eax +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ebx ; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %edi -; X86-NOSSE-NEXT: movl %edi, 8(%esi) -; X86-NOSSE-NEXT: movl %edx, 12(%esi) -; X86-NOSSE-NEXT: movl %eax, (%esi) -; X86-NOSSE-NEXT: movl %ecx, 4(%esi) -; X86-NOSSE-NEXT: movl %esi, %eax -; X86-NOSSE-NEXT: addl $20, %esp +; X86-NOSSE-NEXT: movl (%esi), %eax +; X86-NOSSE-NEXT: movl 4(%esi), %edx +; X86-NOSSE-NEXT: .p2align 4, 0x90 +; X86-NOSSE-NEXT: .LBB8_1: # %atomicrmw.start +; X86-NOSSE-NEXT: # =>This Inner Loop Header: Depth=1 +; X86-NOSSE-NEXT: lock cmpxchg8b (%esi) +; X86-NOSSE-NEXT: jne .LBB8_1 +; X86-NOSSE-NEXT: # %bb.2: # %atomicrmw.end +; X86-NOSSE-NEXT: movl %eax, (%esp) +; X86-NOSSE-NEXT: movl %edx, {{[0-9]+}}(%esp) +; X86-NOSSE-NEXT: fldl (%esp) +; X86-NOSSE-NEXT: addl $12, %esp ; X86-NOSSE-NEXT: .cfi_def_cfa_offset 12 ; X86-NOSSE-NEXT: popl %esi ; X86-NOSSE-NEXT: .cfi_def_cfa_offset 8 -; X86-NOSSE-NEXT: popl %edi +; X86-NOSSE-NEXT: popl %ebx ; X86-NOSSE-NEXT: .cfi_def_cfa_offset 4 -; X86-NOSSE-NEXT: retl $4 +; X86-NOSSE-NEXT: retl ; -; X64-SSE-LABEL: load_fp128: +; X64-SSE-LABEL: exchange_double: ; X64-SSE: # %bb.0: -; X64-SSE-NEXT: subq $24, %rsp -; X64-SSE-NEXT: .cfi_def_cfa_offset 32 -; X64-SSE-NEXT: xorl %esi, %esi -; X64-SSE-NEXT: xorl %edx, %edx -; X64-SSE-NEXT: xorl %ecx, %ecx -; X64-SSE-NEXT: xorl %r8d, %r8d -; X64-SSE-NEXT: callq __sync_val_compare_and_swap_16@PLT -; X64-SSE-NEXT: movq %rdx, {{[0-9]+}}(%rsp) -; X64-SSE-NEXT: movq %rax, (%rsp) -; X64-SSE-NEXT: movaps (%rsp), %xmm0 -; X64-SSE-NEXT: addq $24, %rsp -; X64-SSE-NEXT: .cfi_def_cfa_offset 8 +; X64-SSE-NEXT: movq %xmm0, %rax +; X64-SSE-NEXT: xchgq %rax, (%rdi) +; X64-SSE-NEXT: movq %rax, %xmm0 ; X64-SSE-NEXT: retq ; -; X64-AVX-LABEL: load_fp128: +; X64-AVX-LABEL: exchange_double: ; X64-AVX: # %bb.0: -; X64-AVX-NEXT: subq $24, %rsp -; X64-AVX-NEXT: .cfi_def_cfa_offset 32 -; X64-AVX-NEXT: xorl %esi, %esi -; X64-AVX-NEXT: xorl %edx, %edx -; X64-AVX-NEXT: xorl %ecx, %ecx -; X64-AVX-NEXT: xorl %r8d, %r8d -; X64-AVX-NEXT: callq __sync_val_compare_and_swap_16@PLT -; X64-AVX-NEXT: movq %rdx, {{[0-9]+}}(%rsp) -; X64-AVX-NEXT: movq %rax, (%rsp) -; X64-AVX-NEXT: vmovaps (%rsp), %xmm0 -; X64-AVX-NEXT: addq $24, %rsp -; X64-AVX-NEXT: .cfi_def_cfa_offset 8 +; X64-AVX-NEXT: vmovq %xmm0, %rax +; X64-AVX-NEXT: xchgq %rax, (%rdi) +; X64-AVX-NEXT: vmovq %rax, %xmm0 ; X64-AVX-NEXT: retq - %v = load atomic fp128, ptr %fptr unordered, align 16 - ret fp128 %v + %v = atomicrmw xchg ptr %fptr, double %x monotonic, align 8 + ret double %v } diff --git a/llvm/test/CodeGen/X86/atomic128.ll b/llvm/test/CodeGen/X86/atomic128.ll index d5600b54a169d2fc5be663329dd9de856ef11179..1f7c2254bc79fa5eb1a46a3add54ef8ec9f4b377 100644 --- a/llvm/test/CodeGen/X86/atomic128.ll +++ b/llvm/test/CodeGen/X86/atomic128.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-apple-macosx10.9 -verify-machineinstrs -mattr=cx16 | FileCheck %s -; RUN: llc < %s -mtriple=i386-linux-gnu -verify-machineinstrs -mattr=cx16 | FileCheck %s -check-prefixes=CHECK32 -; RUN: llc < %s -mtriple=i386-linux-gnu -verify-machineinstrs -mattr=-cx16 | FileCheck %s -check-prefixes=CHECK32 + +; Codegen of i128 without cx16 is tested in atomic-nocx16.ll @var = global i128 0 @@ -20,61 +20,6 @@ define i128 @val_compare_and_swap(ptr %p, i128 %oldval, i128 %newval) { ; CHECK-NEXT: lock cmpxchg16b (%rdi) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: val_compare_and_swap: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %edi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 12 -; CHECK32-NEXT: subl $20, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -12 -; CHECK32-NEXT: .cfi_offset %edi, -8 -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_val_compare_and_swap_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $44, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -44 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edi -; CHECK32-NEXT: movl %edi, 8(%esi) -; CHECK32-NEXT: movl %edx, 12(%esi) -; CHECK32-NEXT: movl %eax, (%esi) -; CHECK32-NEXT: movl %ecx, 4(%esi) -; CHECK32-NEXT: movl %esi, %eax -; CHECK32-NEXT: addl $20, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 12 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %edi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl $4 %pair = cmpxchg ptr %p, i128 %oldval, i128 %newval acquire acquire %val = extractvalue { i128, i1 } %pair, 0 ret i128 %val @@ -94,24 +39,6 @@ define void @cmpxchg16b_global_with_offset() nounwind { ; CHECK-NEXT: lock cmpxchg16b _cmpxchg16b_global+16(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: cmpxchg16b_global_with_offset: -; CHECK32: # %bb.0: # %entry -; CHECK32-NEXT: subl $36, %esp -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $cmpxchg16b_global+16 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: calll __sync_val_compare_and_swap_16 -; CHECK32-NEXT: addl $72, %esp -; CHECK32-NEXT: retl entry: %0 = load atomic i128, ptr getelementptr inbounds ({i128, i128}, ptr @cmpxchg16b_global, i64 0, i32 1) acquire, align 16 ret void @@ -142,46 +69,6 @@ define void @fetch_and_nand(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_nand: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_nand_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw nand ptr %p, i128 %bits release store i128 %val, ptr @var, align 16 ret void @@ -210,46 +97,6 @@ define void @fetch_and_or(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_or: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_or_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw or ptr %p, i128 %bits seq_cst store i128 %val, ptr @var, align 16 ret void @@ -278,46 +125,6 @@ define void @fetch_and_add(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_add: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_add_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw add ptr %p, i128 %bits seq_cst store i128 %val, ptr @var, align 16 ret void @@ -346,46 +153,6 @@ define void @fetch_and_sub(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_sub: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_sub_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw sub ptr %p, i128 %bits seq_cst store i128 %val, ptr @var, align 16 ret void @@ -417,46 +184,6 @@ define void @fetch_and_min(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_min: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_min_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw min ptr %p, i128 %bits seq_cst store i128 %val, ptr @var, align 16 ret void @@ -488,46 +215,6 @@ define void @fetch_and_max(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_max: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_max_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw max ptr %p, i128 %bits seq_cst store i128 %val, ptr @var, align 16 ret void @@ -559,46 +246,6 @@ define void @fetch_and_umin(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_umin: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_umin_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw umin ptr %p, i128 %bits seq_cst store i128 %val, ptr @var, align 16 ret void @@ -630,46 +277,6 @@ define void @fetch_and_umax(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_umax: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_umax_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw umax ptr %p, i128 %bits seq_cst store i128 %val, ptr @var, align 16 ret void @@ -688,61 +295,6 @@ define i128 @atomic_load_seq_cst(ptr %p) { ; CHECK-NEXT: lock cmpxchg16b (%rdi) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: atomic_load_seq_cst: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %edi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 12 -; CHECK32-NEXT: subl $20, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -12 -; CHECK32-NEXT: .cfi_offset %edi, -8 -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_val_compare_and_swap_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $44, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -44 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edi -; CHECK32-NEXT: movl %edi, 8(%esi) -; CHECK32-NEXT: movl %edx, 12(%esi) -; CHECK32-NEXT: movl %eax, (%esi) -; CHECK32-NEXT: movl %ecx, 4(%esi) -; CHECK32-NEXT: movl %esi, %eax -; CHECK32-NEXT: addl $20, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 12 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %edi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl $4 %r = load atomic i128, ptr %p seq_cst, align 16 ret i128 %r } @@ -760,61 +312,6 @@ define i128 @atomic_load_relaxed(ptr %p) { ; CHECK-NEXT: lock cmpxchg16b (%rdi) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: atomic_load_relaxed: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %edi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 12 -; CHECK32-NEXT: subl $20, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -12 -; CHECK32-NEXT: .cfi_offset %edi, -8 -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_val_compare_and_swap_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $44, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -44 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edi -; CHECK32-NEXT: movl %edi, 8(%esi) -; CHECK32-NEXT: movl %edx, 12(%esi) -; CHECK32-NEXT: movl %eax, (%esi) -; CHECK32-NEXT: movl %ecx, 4(%esi) -; CHECK32-NEXT: movl %esi, %eax -; CHECK32-NEXT: addl $20, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 12 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %edi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl $4 %r = load atomic i128, ptr %p monotonic, align 16 ret i128 %r } @@ -837,29 +334,6 @@ define void @atomic_store_seq_cst(ptr %p, i128 %in) { ; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: atomic_store_seq_cst: -; CHECK32: # %bb.0: -; CHECK32-NEXT: subl $36, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 36 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_lock_test_and_set_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $56, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -56 -; CHECK32-NEXT: retl store atomic i128 %in, ptr %p seq_cst, align 16 ret void } @@ -882,29 +356,6 @@ define void @atomic_store_release(ptr %p, i128 %in) { ; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: atomic_store_release: -; CHECK32: # %bb.0: -; CHECK32-NEXT: subl $36, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 36 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_lock_test_and_set_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $56, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -56 -; CHECK32-NEXT: retl store atomic i128 %in, ptr %p release, align 16 ret void } @@ -927,29 +378,6 @@ define void @atomic_store_relaxed(ptr %p, i128 %in) { ; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: atomic_store_relaxed: -; CHECK32: # %bb.0: -; CHECK32-NEXT: subl $36, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 36 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_lock_test_and_set_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $56, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -56 -; CHECK32-NEXT: retl store atomic i128 %in, ptr %p unordered, align 16 ret void } diff --git a/llvm/test/CodeGen/X86/atomicf128.ll b/llvm/test/CodeGen/X86/atomicf128.ll deleted file mode 100644 index 3b0bba403aa5d504628a0704e5106cb2a130f270..0000000000000000000000000000000000000000 --- a/llvm/test/CodeGen/X86/atomicf128.ll +++ /dev/null @@ -1,46 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-apple-macosx10.9 -verify-machineinstrs -mattr=cx16 | FileCheck %s -; RUN: llc < %s -mtriple=x86_64-apple-macosx10.9 -verify-machineinstrs -mattr=cx16 -mattr=-sse | FileCheck %s --check-prefix=NOSSE - -; FIXME: This test has a fatal error in 32-bit mode - -@fsc128 = external global fp128 - -define void @atomic_fetch_swapf128(fp128 %x) nounwind { -; CHECK-LABEL: atomic_fetch_swapf128: -; CHECK: ## %bb.0: -; CHECK-NEXT: pushq %rbx -; CHECK-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp) -; CHECK-NEXT: movq -{{[0-9]+}}(%rsp), %rbx -; CHECK-NEXT: movq -{{[0-9]+}}(%rsp), %rcx -; CHECK-NEXT: movq _fsc128@GOTPCREL(%rip), %rsi -; CHECK-NEXT: movq (%rsi), %rax -; CHECK-NEXT: movq 8(%rsi), %rdx -; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: LBB0_1: ## %atomicrmw.start -; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: lock cmpxchg16b (%rsi) -; CHECK-NEXT: jne LBB0_1 -; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end -; CHECK-NEXT: popq %rbx -; CHECK-NEXT: retq -; -; NOSSE-LABEL: atomic_fetch_swapf128: -; NOSSE: ## %bb.0: -; NOSSE-NEXT: pushq %rbx -; NOSSE-NEXT: movq %rsi, %rcx -; NOSSE-NEXT: movq %rdi, %rbx -; NOSSE-NEXT: movq _fsc128@GOTPCREL(%rip), %rsi -; NOSSE-NEXT: movq (%rsi), %rax -; NOSSE-NEXT: movq 8(%rsi), %rdx -; NOSSE-NEXT: .p2align 4, 0x90 -; NOSSE-NEXT: LBB0_1: ## %atomicrmw.start -; NOSSE-NEXT: ## =>This Inner Loop Header: Depth=1 -; NOSSE-NEXT: lock cmpxchg16b (%rsi) -; NOSSE-NEXT: jne LBB0_1 -; NOSSE-NEXT: ## %bb.2: ## %atomicrmw.end -; NOSSE-NEXT: popq %rbx -; NOSSE-NEXT: retq - %t1 = atomicrmw xchg ptr @fsc128, fp128 %x acquire - ret void -} diff --git a/llvm/test/CodeGen/X86/avx512fp16-arith.ll b/llvm/test/CodeGen/X86/avx512fp16-arith.ll index 77cab3d1512cae03881249fd2976ff70ac50e896..8d811d8d29e065629184312043e2bae210f635a7 100644 --- a/llvm/test/CodeGen/X86/avx512fp16-arith.ll +++ b/llvm/test/CodeGen/X86/avx512fp16-arith.ll @@ -329,7 +329,7 @@ define half @fcopysign(half %x, half %y) { ; CHECK-LABEL: fcopysign: ; CHECK: ## %bb.0: ; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] -; CHECK-NEXT: vpternlogq $226, %xmm1, %xmm2, %xmm0 +; CHECK-NEXT: vpternlogd $226, %xmm1, %xmm2, %xmm0 ; CHECK-NEXT: retq %a = call half @llvm.copysign.f16(half %x, half %y) ret half %a @@ -384,7 +384,7 @@ declare <8 x half> @llvm.fabs.v8f16(<8 x half>) define <8 x half> @fcopysignv8f16(<8 x half> %x, <8 x half> %y) { ; CHECK-LABEL: fcopysignv8f16: ; CHECK: ## %bb.0: -; CHECK-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; CHECK-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; CHECK-NEXT: retq %a = call <8 x half> @llvm.copysign.v8f16(<8 x half> %x, <8 x half> %y) ret <8 x half> %a @@ -439,7 +439,7 @@ declare <16 x half> @llvm.fabs.v16f16(<16 x half>) define <16 x half> @fcopysignv16f16(<16 x half> %x, <16 x half> %y) { ; CHECK-LABEL: fcopysignv16f16: ; CHECK: ## %bb.0: -; CHECK-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; CHECK-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; CHECK-NEXT: retq %a = call <16 x half> @llvm.copysign.v16f16(<16 x half> %x, <16 x half> %y) ret <16 x half> %a @@ -494,7 +494,7 @@ declare <32 x half> @llvm.fabs.v32f16(<32 x half>) define <32 x half> @fcopysignv32f16(<32 x half> %x, <32 x half> %y) { ; CHECK-LABEL: fcopysignv32f16: ; CHECK: ## %bb.0: -; CHECK-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; CHECK-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; CHECK-NEXT: retq %a = call <32 x half> @llvm.copysign.v32f16(<32 x half> %x, <32 x half> %y) ret <32 x half> %a diff --git a/llvm/test/CodeGen/X86/code-model-elf.ll b/llvm/test/CodeGen/X86/code-model-elf.ll index 483ffd87ac696f2b32231dfe8f537a5518240007..457c6bde354f2e22c21ae91fd8e5787aab05b577 100644 --- a/llvm/test/CodeGen/X86/code-model-elf.ll +++ b/llvm/test/CodeGen/X86/code-model-elf.ll @@ -41,7 +41,8 @@ target triple = "x86_64--linux" @extern_data = external global [10 x i32], align 16 @thread_data = external thread_local global i32, align 4 @unknown_size_data = dso_local global [0 x i32] zeroinitializer, align 16 - +@forced_small_data = dso_local global [10 x i32] zeroinitializer, code_model "small", align 16 +@forced_large_data = dso_local global [10 x i32] zeroinitializer, code_model "large", align 16 define dso_local ptr @lea_static_data() #0 { ; SMALL-STATIC-LABEL: lea_static_data: @@ -223,6 +224,199 @@ define dso_local ptr @lea_unknown_size_data() #0 { ret ptr @unknown_size_data } +; TODO: make nopic small and medium instruction sequence the same +define dso_local ptr @lea_forced_small_data() #0 { +; SMALL-STATIC-LABEL: lea_forced_small_data: +; SMALL-STATIC: # %bb.0: +; SMALL-STATIC-NEXT: movl $forced_small_data, %eax +; SMALL-STATIC-NEXT: retq +; +; MEDIUM-STATIC-LABEL: lea_forced_small_data: +; MEDIUM-STATIC: # %bb.0: +; MEDIUM-STATIC-NEXT: movabsq $forced_small_data, %rax +; MEDIUM-STATIC-NEXT: retq +; +; LARGE-STATIC-LABEL: lea_forced_small_data: +; LARGE-STATIC: # %bb.0: +; LARGE-STATIC-NEXT: movabsq $forced_small_data, %rax +; LARGE-STATIC-NEXT: retq +; +; SMALL-PIC-LABEL: lea_forced_small_data: +; SMALL-PIC: # %bb.0: +; SMALL-PIC-NEXT: leaq forced_small_data(%rip), %rax +; SMALL-PIC-NEXT: retq +; +; MEDIUM-SMALL-DATA-PIC-LABEL: lea_forced_small_data: +; MEDIUM-SMALL-DATA-PIC: # %bb.0: +; MEDIUM-SMALL-DATA-PIC-NEXT: leaq forced_small_data(%rip), %rax +; MEDIUM-SMALL-DATA-PIC-NEXT: retq +; +; MEDIUM-PIC-LABEL: lea_forced_small_data: +; MEDIUM-PIC: # %bb.0: +; MEDIUM-PIC-NEXT: leaq forced_small_data(%rip), %rax +; MEDIUM-PIC-NEXT: retq +; +; LARGE-PIC-LABEL: lea_forced_small_data: +; LARGE-PIC: # %bb.0: +; LARGE-PIC-NEXT: .L4$pb: +; LARGE-PIC-NEXT: leaq .L4$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L4$pb, %rcx +; LARGE-PIC-NEXT: addq %rax, %rcx +; LARGE-PIC-NEXT: movabsq $forced_small_data@GOTOFF, %rax +; LARGE-PIC-NEXT: addq %rcx, %rax +; LARGE-PIC-NEXT: retq + ret ptr @forced_small_data +} + +; TODO: make small and medium instruction sequence the same +define dso_local i32 @load_forced_small_data() #0 { +; SMALL-STATIC-LABEL: load_forced_small_data: +; SMALL-STATIC: # %bb.0: +; SMALL-STATIC-NEXT: movl forced_small_data+8(%rip), %eax +; SMALL-STATIC-NEXT: retq +; +; MEDIUM-STATIC-LABEL: load_forced_small_data: +; MEDIUM-STATIC: # %bb.0: +; MEDIUM-STATIC-NEXT: movabsq $forced_small_data, %rax +; MEDIUM-STATIC-NEXT: movl 8(%rax), %eax +; MEDIUM-STATIC-NEXT: retq +; +; LARGE-STATIC-LABEL: load_forced_small_data: +; LARGE-STATIC: # %bb.0: +; LARGE-STATIC-NEXT: movabsq $forced_small_data, %rax +; LARGE-STATIC-NEXT: movl 8(%rax), %eax +; LARGE-STATIC-NEXT: retq +; +; SMALL-PIC-LABEL: load_forced_small_data: +; SMALL-PIC: # %bb.0: +; SMALL-PIC-NEXT: movl forced_small_data+8(%rip), %eax +; SMALL-PIC-NEXT: retq +; +; MEDIUM-SMALL-DATA-PIC-LABEL: load_forced_small_data: +; MEDIUM-SMALL-DATA-PIC: # %bb.0: +; MEDIUM-SMALL-DATA-PIC-NEXT: leaq forced_small_data(%rip), %rax +; MEDIUM-SMALL-DATA-PIC-NEXT: movl 8(%rax), %eax +; MEDIUM-SMALL-DATA-PIC-NEXT: retq +; +; MEDIUM-PIC-LABEL: load_forced_small_data: +; MEDIUM-PIC: # %bb.0: +; MEDIUM-PIC-NEXT: leaq forced_small_data(%rip), %rax +; MEDIUM-PIC-NEXT: movl 8(%rax), %eax +; MEDIUM-PIC-NEXT: retq +; +; LARGE-PIC-LABEL: load_forced_small_data: +; LARGE-PIC: # %bb.0: +; LARGE-PIC-NEXT: .L5$pb: +; LARGE-PIC-NEXT: leaq .L5$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L5$pb, %rcx +; LARGE-PIC-NEXT: addq %rax, %rcx +; LARGE-PIC-NEXT: movabsq $forced_small_data@GOTOFF, %rax +; LARGE-PIC-NEXT: movl 8(%rcx,%rax), %eax +; LARGE-PIC-NEXT: retq + %rv = load i32, ptr getelementptr inbounds (i32, ptr @forced_small_data, i64 2) + ret i32 %rv +} + +; TODO: fix small code model instruction sequences to use 64-bit constants +define dso_local ptr @lea_forced_large_data() #0 { +; SMALL-STATIC-LABEL: lea_forced_large_data: +; SMALL-STATIC: # %bb.0: +; SMALL-STATIC-NEXT: movl $forced_large_data, %eax +; SMALL-STATIC-NEXT: retq +; +; MEDIUM-STATIC-LABEL: lea_forced_large_data: +; MEDIUM-STATIC: # %bb.0: +; MEDIUM-STATIC-NEXT: movabsq $forced_large_data, %rax +; MEDIUM-STATIC-NEXT: retq +; +; LARGE-STATIC-LABEL: lea_forced_large_data: +; LARGE-STATIC: # %bb.0: +; LARGE-STATIC-NEXT: movabsq $forced_large_data, %rax +; LARGE-STATIC-NEXT: retq +; +; SMALL-PIC-LABEL: lea_forced_large_data: +; SMALL-PIC: # %bb.0: +; SMALL-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax +; SMALL-PIC-NEXT: leaq forced_large_data@GOTOFF(%rax), %rax +; SMALL-PIC-NEXT: retq +; +; MEDIUM-SMALL-DATA-PIC-LABEL: lea_forced_large_data: +; MEDIUM-SMALL-DATA-PIC: # %bb.0: +; MEDIUM-SMALL-DATA-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rcx +; MEDIUM-SMALL-DATA-PIC-NEXT: movabsq $forced_large_data@GOTOFF, %rax +; MEDIUM-SMALL-DATA-PIC-NEXT: addq %rcx, %rax +; MEDIUM-SMALL-DATA-PIC-NEXT: retq +; +; MEDIUM-PIC-LABEL: lea_forced_large_data: +; MEDIUM-PIC: # %bb.0: +; MEDIUM-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rcx +; MEDIUM-PIC-NEXT: movabsq $forced_large_data@GOTOFF, %rax +; MEDIUM-PIC-NEXT: addq %rcx, %rax +; MEDIUM-PIC-NEXT: retq +; +; LARGE-PIC-LABEL: lea_forced_large_data: +; LARGE-PIC: # %bb.0: +; LARGE-PIC-NEXT: .L6$pb: +; LARGE-PIC-NEXT: leaq .L6$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L6$pb, %rcx +; LARGE-PIC-NEXT: addq %rax, %rcx +; LARGE-PIC-NEXT: movabsq $forced_large_data@GOTOFF, %rax +; LARGE-PIC-NEXT: addq %rcx, %rax +; LARGE-PIC-NEXT: retq + ret ptr @forced_large_data +} + +define dso_local i32 @load_forced_large_data() #0 { +; SMALL-STATIC-LABEL: load_forced_large_data: +; SMALL-STATIC: # %bb.0: +; SMALL-STATIC-NEXT: movl forced_large_data+8(%rip), %eax +; SMALL-STATIC-NEXT: retq +; +; MEDIUM-STATIC-LABEL: load_forced_large_data: +; MEDIUM-STATIC: # %bb.0: +; MEDIUM-STATIC-NEXT: movabsq $forced_large_data, %rax +; MEDIUM-STATIC-NEXT: movl 8(%rax), %eax +; MEDIUM-STATIC-NEXT: retq +; +; LARGE-STATIC-LABEL: load_forced_large_data: +; LARGE-STATIC: # %bb.0: +; LARGE-STATIC-NEXT: movabsq $forced_large_data, %rax +; LARGE-STATIC-NEXT: movl 8(%rax), %eax +; LARGE-STATIC-NEXT: retq +; +; SMALL-PIC-LABEL: load_forced_large_data: +; SMALL-PIC: # %bb.0: +; SMALL-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax +; SMALL-PIC-NEXT: movl forced_large_data@GOTOFF+8(%rax), %eax +; SMALL-PIC-NEXT: retq +; +; MEDIUM-SMALL-DATA-PIC-LABEL: load_forced_large_data: +; MEDIUM-SMALL-DATA-PIC: # %bb.0: +; MEDIUM-SMALL-DATA-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax +; MEDIUM-SMALL-DATA-PIC-NEXT: movabsq $forced_large_data@GOTOFF, %rcx +; MEDIUM-SMALL-DATA-PIC-NEXT: movl 8(%rax,%rcx), %eax +; MEDIUM-SMALL-DATA-PIC-NEXT: retq +; +; MEDIUM-PIC-LABEL: load_forced_large_data: +; MEDIUM-PIC: # %bb.0: +; MEDIUM-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax +; MEDIUM-PIC-NEXT: movabsq $forced_large_data@GOTOFF, %rcx +; MEDIUM-PIC-NEXT: movl 8(%rax,%rcx), %eax +; MEDIUM-PIC-NEXT: retq +; +; LARGE-PIC-LABEL: load_forced_large_data: +; LARGE-PIC: # %bb.0: +; LARGE-PIC-NEXT: .L7$pb: +; LARGE-PIC-NEXT: leaq .L7$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L7$pb, %rcx +; LARGE-PIC-NEXT: addq %rax, %rcx +; LARGE-PIC-NEXT: movabsq $forced_large_data@GOTOFF, %rax +; LARGE-PIC-NEXT: movl 8(%rcx,%rax), %eax +; LARGE-PIC-NEXT: retq + %rv = load i32, ptr getelementptr inbounds (i32, ptr @forced_large_data, i64 2) + ret i32 %rv +} + define dso_local i32 @load_global_data() #0 { ; SMALL-STATIC-LABEL: load_global_data: ; SMALL-STATIC: # %bb.0: @@ -261,9 +455,9 @@ define dso_local i32 @load_global_data() #0 { ; ; LARGE-PIC-LABEL: load_global_data: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L4$pb: -; LARGE-PIC-NEXT: leaq .L4$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L4$pb, %rcx +; LARGE-PIC-NEXT: .L8$pb: +; LARGE-PIC-NEXT: leaq .L8$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L8$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $global_data@GOTOFF, %rax ; LARGE-PIC-NEXT: movl 8(%rcx,%rax), %eax @@ -311,9 +505,9 @@ define dso_local i32 @load_extern_data() #0 { ; ; LARGE-PIC-LABEL: load_extern_data: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L5$pb: -; LARGE-PIC-NEXT: leaq .L5$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L5$pb, %rcx +; LARGE-PIC-NEXT: .L9$pb: +; LARGE-PIC-NEXT: leaq .L9$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L9$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $extern_data@GOT, %rax ; LARGE-PIC-NEXT: movq (%rcx,%rax), %rax @@ -362,9 +556,9 @@ define dso_local i32 @load_unknown_size_data() #0 { ; ; LARGE-PIC-LABEL: load_unknown_size_data: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L6$pb: -; LARGE-PIC-NEXT: leaq .L6$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L6$pb, %rcx +; LARGE-PIC-NEXT: .L10$pb: +; LARGE-PIC-NEXT: leaq .L10$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L10$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $unknown_size_data@GOTOFF, %rax ; LARGE-PIC-NEXT: movl 8(%rcx,%rax), %eax @@ -422,9 +616,9 @@ define dso_local ptr @lea_static_fn() #0 { ; ; LARGE-PIC-LABEL: lea_static_fn: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L9$pb: -; LARGE-PIC-NEXT: leaq .L9$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L9$pb, %rcx +; LARGE-PIC-NEXT: .L13$pb: +; LARGE-PIC-NEXT: leaq .L13$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L13$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $static_fn@GOTOFF, %rax ; LARGE-PIC-NEXT: addq %rcx, %rax @@ -465,9 +659,9 @@ define dso_local ptr @lea_global_fn() #0 { ; ; LARGE-PIC-LABEL: lea_global_fn: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L10$pb: -; LARGE-PIC-NEXT: leaq .L10$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L10$pb, %rcx +; LARGE-PIC-NEXT: .L14$pb: +; LARGE-PIC-NEXT: leaq .L14$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L14$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $global_fn@GOTOFF, %rax ; LARGE-PIC-NEXT: addq %rcx, %rax @@ -508,9 +702,9 @@ define dso_local ptr @lea_extern_fn() #0 { ; ; LARGE-PIC-LABEL: lea_extern_fn: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L11$pb: -; LARGE-PIC-NEXT: leaq .L11$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L11$pb, %rcx +; LARGE-PIC-NEXT: .L15$pb: +; LARGE-PIC-NEXT: leaq .L15$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L15$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $extern_fn@GOT, %rax ; LARGE-PIC-NEXT: movq (%rcx,%rax), %rax @@ -586,9 +780,9 @@ define dso_local float @load_constant_pool(float %x) #0 { ; ; LARGE-PIC-LABEL: load_constant_pool: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L13$pb: -; LARGE-PIC-NEXT: leaq .L13$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L13$pb, %rcx +; LARGE-PIC-NEXT: .L17$pb: +; LARGE-PIC-NEXT: leaq .L17$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L17$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq ${{\.?LCPI[0-9]+_[0-9]+}}@GOTOFF, %rax ; LARGE-PIC-NEXT: addss (%rcx,%rax), %xmm0 diff --git a/llvm/test/CodeGen/X86/combine-concatvectors.ll b/llvm/test/CodeGen/X86/combine-concatvectors.ll index a8c3fb589bbfe98ebec9549db5151c9f13618064..31eaa1b205aac96baefc77616977c45de7e8ae78 100644 --- a/llvm/test/CodeGen/X86/combine-concatvectors.ll +++ b/llvm/test/CodeGen/X86/combine-concatvectors.ll @@ -48,8 +48,7 @@ define void @concat_of_broadcast_v2f64_v4f64() { ; AVX1-NEXT: movl $1091567616, 30256(%rax) # imm = 0x41100000 ; AVX1-NEXT: movabsq $4294967297, %rcx # imm = 0x100000001 ; AVX1-NEXT: movq %rcx, 46348(%rax) -; AVX1-NEXT: vbroadcastf128 {{.*#+}} ymm0 = [7.812501848093234E-3,7.812501848093234E-3,7.812501848093234E-3,7.812501848093234E-3] -; AVX1-NEXT: # ymm0 = mem[0,1,0,1] +; AVX1-NEXT: vbroadcastss {{.*#+}} ymm0 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] ; AVX1-NEXT: vmovups %ymm0, 48296(%rax) ; AVX1-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero ; AVX1-NEXT: vmovsd %xmm0, 47372(%rax) diff --git a/llvm/test/CodeGen/X86/dwarf-eh-prepare-dbg.ll b/llvm/test/CodeGen/X86/dwarf-eh-prepare-dbg.ll index 651e196a4ed76ec814159ebe735ff23aa92060da..0345b736e9e60b69f840830eed9648d11f195893 100644 --- a/llvm/test/CodeGen/X86/dwarf-eh-prepare-dbg.ll +++ b/llvm/test/CodeGen/X86/dwarf-eh-prepare-dbg.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=x86_64-linux-gnu -dwarfehprepare < %s | FileCheck %s -; RUN: opt -S -mtriple=x86_64-linux-gnu -passes=dwarfehprepare < %s | FileCheck %s +; RUN: opt -S -mtriple=x86_64-linux-gnu -dwarf-eh-prepare < %s | FileCheck %s +; RUN: opt -S -mtriple=x86_64-linux-gnu -passes=dwarf-eh-prepare < %s | FileCheck %s ; PR57469: If _Unwind_Resume is defined in the same module and we have debug ; info, then the inserted _Unwind_Resume calls also need to have a dummy debug diff --git a/llvm/test/CodeGen/X86/dwarf-eh-prepare.ll b/llvm/test/CodeGen/X86/dwarf-eh-prepare.ll index 6778601cfbf6d0da540c179f1735c7a970044ba7..dd27e7fe141173a30a7d5a55dae2b7bbbee0aedd 100644 --- a/llvm/test/CodeGen/X86/dwarf-eh-prepare.ll +++ b/llvm/test/CodeGen/X86/dwarf-eh-prepare.ll @@ -1,5 +1,5 @@ -; RUN: opt -mtriple=x86_64-linux-gnu -dwarfehprepare -simplifycfg-require-and-preserve-domtree=1 -run-twice < %s -S | FileCheck %s -; RUN: opt -mtriple=x86_64-linux-gnu -passes=dwarfehprepare -codegen-opt-level=2 -simplifycfg-require-and-preserve-domtree=1 -run-twice < %s -S | FileCheck %s +; RUN: opt -mtriple=x86_64-linux-gnu -dwarf-eh-prepare -simplifycfg-require-and-preserve-domtree=1 -run-twice < %s -S | FileCheck %s +; RUN: opt -mtriple=x86_64-linux-gnu -passes=dwarf-eh-prepare -codegen-opt-level=2 -simplifycfg-require-and-preserve-domtree=1 -run-twice < %s -S | FileCheck %s ; Check basic functionality of IR-to-IR DWARF EH preparation. This should ; eliminate resumes. This pass requires a TargetMachine, so we put it under X86 diff --git a/llvm/test/CodeGen/X86/dwarf_eh_resume.ll b/llvm/test/CodeGen/X86/dwarf_eh_resume.ll index 90fad00b68bd81c20dcedce7325deff40c3bd7ba..d66d89d16a394cae33d3eb2abb7c1fe680941f24 100644 --- a/llvm/test/CodeGen/X86/dwarf_eh_resume.ll +++ b/llvm/test/CodeGen/X86/dwarf_eh_resume.ll @@ -1,5 +1,5 @@ -; RUN: opt -mtriple=x86_64-linux-gnu -dwarfehprepare -S %s | FileCheck %s -; RUN: opt -mtriple=x86_64-linux-gnu -passes=dwarfehprepare -S %s | FileCheck %s +; RUN: opt -mtriple=x86_64-linux-gnu -dwarf-eh-prepare -S %s | FileCheck %s +; RUN: opt -mtriple=x86_64-linux-gnu -passes=dwarf-eh-prepare -S %s | FileCheck %s declare i32 @hoge(...) diff --git a/llvm/test/CodeGen/X86/evex-to-vex-compress.mir b/llvm/test/CodeGen/X86/evex-to-vex-compress.mir index 9c49ff8e573e9831750d5e1a880537bfe87b0249..06d3c1532c3eaa03b4e247e9296fb05dffa4b914 100644 --- a/llvm/test/CodeGen/X86/evex-to-vex-compress.mir +++ b/llvm/test/CodeGen/X86/evex-to-vex-compress.mir @@ -4,8 +4,8 @@ --- | define void @evex_z256_to_vex_test() { ret void } - define void @evex_z128_to_vex_test() { ret void } - define void @evex_scalar_to_vex_test() { ret void } + define void @evex_z128_to_vex_test() { ret void } + define void @evex_scalar_to_vex_test() { ret void } define void @evex_z256_to_evex_test() { ret void } define void @evex_z128_to_evex_test() { ret void } define void @evex_scalar_to_evex_test() { ret void } @@ -14,141 +14,141 @@ # CHECK-LABEL: name: evex_z256_to_vex_test # CHECK: bb.0: -name: evex_z256_to_vex_test -body: | - bb.0: +name: evex_z256_to_vex_test +body: | + bb.0: ; CHECK: VMOVAPDYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVAPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVAPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVAPDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVAPDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVAPDYrr $ymm0 - $ymm0 = VMOVAPDZ256rr $ymm0 + $ymm0 = VMOVAPDZ256rr $ymm0 ; CHECK: VMOVAPSYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVAPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVAPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVAPSYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVAPSZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVAPSYrr $ymm0 - $ymm0 = VMOVAPSZ256rr $ymm0 + $ymm0 = VMOVAPSZ256rr $ymm0 ; CHECK: $ymm0 = VMOVDDUPYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVDDUPZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVDDUPYrr $ymm0 - $ymm0 = VMOVDDUPZ256rr $ymm0 + $ymm0 = VMOVDDUPZ256rr $ymm0 ; CHECK: VMOVDQAYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVDQA32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVDQA32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVDQAYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVDQA32Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVDQAYrr $ymm0 - $ymm0 = VMOVDQA32Z256rr $ymm0 + $ymm0 = VMOVDQA32Z256rr $ymm0 ; CHECK: VMOVDQAYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVDQA64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVDQA64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVDQAYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVDQA64Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVDQAYrr $ymm0 - $ymm0 = VMOVDQA64Z256rr $ymm0 + $ymm0 = VMOVDQA64Z256rr $ymm0 ; CHECK: VMOVDQUYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVDQU16Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVDQU16Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVDQUYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVDQU16Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVDQUYrr $ymm0 - $ymm0 = VMOVDQU16Z256rr $ymm0 + $ymm0 = VMOVDQU16Z256rr $ymm0 ; CHECK: VMOVDQUYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVDQU32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVDQU32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVDQUYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVDQU32Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVDQUYrr $ymm0 - $ymm0 = VMOVDQU32Z256rr $ymm0 + $ymm0 = VMOVDQU32Z256rr $ymm0 ; CHECK: VMOVDQUYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVDQU64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVDQU64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVDQUYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVDQU64Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVDQUYrr $ymm0 - $ymm0 = VMOVDQU64Z256rr $ymm0 + $ymm0 = VMOVDQU64Z256rr $ymm0 ; CHECK: VMOVDQUYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVDQU8Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVDQU8Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVDQUYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVDQU8Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVDQUYrr $ymm0 - $ymm0 = VMOVDQU8Z256rr $ymm0 + $ymm0 = VMOVDQU8Z256rr $ymm0 ; CHECK: $ymm0 = VMOVNTDQAYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVNTDQAZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: VMOVNTDQYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVNTDQZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVNTDQZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: VMOVNTPDYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVNTPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVNTPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: VMOVNTPSYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 VMOVNTPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVSHDUPYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVSHDUPZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVSHDUPYrr $ymm0 - $ymm0 = VMOVSHDUPZ256rr $ymm0 + $ymm0 = VMOVSHDUPZ256rr $ymm0 ; CHECK: $ymm0 = VMOVSLDUPYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVSLDUPZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVSLDUPYrr $ymm0 - $ymm0 = VMOVSLDUPZ256rr $ymm0 + $ymm0 = VMOVSLDUPZ256rr $ymm0 ; CHECK: VMOVUPDYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVUPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVUPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVUPDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVUPDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVUPDYrr $ymm0 - $ymm0 = VMOVUPDZ256rr $ymm0 + $ymm0 = VMOVUPDZ256rr $ymm0 ; CHECK: VMOVUPSYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVUPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVUPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VPANDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPANDDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPANDYrr $ymm0, $ymm1 - $ymm0 = VPANDDZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPANDYrr $ymm0, $ymm1 + $ymm0 = VPANDDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPANDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPANDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPANDYrr $ymm0, $ymm1 - $ymm0 = VPANDQZ256rr $ymm0, $ymm1 + $ymm0 = VPANDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPANDNYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPANDNDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPANDNYrr $ymm0, $ymm1 - $ymm0 = VPANDNDZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPANDNYrr $ymm0, $ymm1 + $ymm0 = VPANDNDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPANDNYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPANDNQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPANDNYrr $ymm0, $ymm1 - $ymm0 = VPANDNQZ256rr $ymm0, $ymm1 + $ymm0 = VPANDNQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPAVGBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPAVGBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPAVGBYrr $ymm0, $ymm1 - $ymm0 = VPAVGBZ256rr $ymm0, $ymm1 + $ymm0 = VPAVGBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPAVGWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPAVGWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPAVGWYrr $ymm0, $ymm1 - $ymm0 = VPAVGWZ256rr $ymm0, $ymm1 + $ymm0 = VPAVGWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPADDBYrr $ymm0, $ymm1 - $ymm0 = VPADDBZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPADDBYrr $ymm0, $ymm1 + $ymm0 = VPADDBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPADDDYrr $ymm0, $ymm1 - $ymm0 = VPADDDZ256rr $ymm0, $ymm1 + $ymm0 = VPADDDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPADDQYrr $ymm0, $ymm1 - $ymm0 = VPADDQZ256rr $ymm0, $ymm1 + $ymm0 = VPADDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDSBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDSBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPADDSBYrr $ymm0, $ymm1 - $ymm0 = VPADDSBZ256rr $ymm0, $ymm1 + $ymm0 = VPADDSBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPADDSWYrr $ymm0, $ymm1 - $ymm0 = VPADDSWZ256rr $ymm0, $ymm1 + $ymm0 = VPADDSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDUSBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDUSBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPADDUSBYrr $ymm0, $ymm1 - $ymm0 = VPADDUSBZ256rr $ymm0, $ymm1 + $ymm0 = VPADDUSBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDUSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDUSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPADDUSWYrr $ymm0, $ymm1 - $ymm0 = VPADDUSWZ256rr $ymm0, $ymm1 + $ymm0 = VPADDUSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPADDWYrr $ymm0, $ymm1 - $ymm0 = VPADDWZ256rr $ymm0, $ymm1 + $ymm0 = VPADDWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VMULPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm0 = VMULPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm0 = VMULPDYrr $ymm0, $ymm1, implicit $mxcsr @@ -160,143 +160,143 @@ body: | ; CHECK: $ymm0 = VORPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VORPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VORPDYrr $ymm0, $ymm1 - $ymm0 = VORPDZ256rr $ymm0, $ymm1 + $ymm0 = VORPDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VORPSYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VORPSZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VORPSYrr $ymm0, $ymm1 - $ymm0 = VORPSZ256rr $ymm0, $ymm1 + $ymm0 = VORPSZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMADDUBSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMADDUBSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMADDUBSWYrr $ymm0, $ymm1 - $ymm0 = VPMADDUBSWZ256rr $ymm0, $ymm1 + $ymm0 = VPMADDUBSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMADDWDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMADDWDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMADDWDYrr $ymm0, $ymm1 - $ymm0 = VPMADDWDZ256rr $ymm0, $ymm1 + $ymm0 = VPMADDWDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMAXSBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMAXSBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMAXSBYrr $ymm0, $ymm1 - $ymm0 = VPMAXSBZ256rr $ymm0, $ymm1 + $ymm0 = VPMAXSBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMAXSDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMAXSDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMAXSDYrr $ymm0, $ymm1 - $ymm0 = VPMAXSDZ256rr $ymm0, $ymm1 + $ymm0 = VPMAXSDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMAXSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMAXSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMAXSWYrr $ymm0, $ymm1 - $ymm0 = VPMAXSWZ256rr $ymm0, $ymm1 + $ymm0 = VPMAXSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMAXUBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMAXUBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMAXUBYrr $ymm0, $ymm1 - $ymm0 = VPMAXUBZ256rr $ymm0, $ymm1 + $ymm0 = VPMAXUBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMAXUDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMAXUDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMAXUDYrr $ymm0, $ymm1 - $ymm0 = VPMAXUDZ256rr $ymm0, $ymm1 + $ymm0 = VPMAXUDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMAXUWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMAXUWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMAXUWYrr $ymm0, $ymm1 - $ymm0 = VPMAXUWZ256rr $ymm0, $ymm1 + $ymm0 = VPMAXUWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMINSBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMINSBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMINSBYrr $ymm0, $ymm1 - $ymm0 = VPMINSBZ256rr $ymm0, $ymm1 + $ymm0 = VPMINSBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMINSDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMINSDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMINSDYrr $ymm0, $ymm1 - $ymm0 = VPMINSDZ256rr $ymm0, $ymm1 + $ymm0 = VPMINSDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMINSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMINSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMINSWYrr $ymm0, $ymm1 - $ymm0 = VPMINSWZ256rr $ymm0, $ymm1 + $ymm0 = VPMINSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMINUBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMINUBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMINUBYrr $ymm0, $ymm1 - $ymm0 = VPMINUBZ256rr $ymm0, $ymm1 + $ymm0 = VPMINUBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMINUDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMINUDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMINUDYrr $ymm0, $ymm1 - $ymm0 = VPMINUDZ256rr $ymm0, $ymm1 + $ymm0 = VPMINUDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMINUWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMINUWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMINUWYrr $ymm0, $ymm1 - $ymm0 = VPMINUWZ256rr $ymm0, $ymm1 + $ymm0 = VPMINUWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMULDQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMULDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPMULDQYrr $ymm0, $ymm1 - $ymm0 = VPMULDQZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPMULDQYrr $ymm0, $ymm1 + $ymm0 = VPMULDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMULHRSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMULHRSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMULHRSWYrr $ymm0, $ymm1 - $ymm0 = VPMULHRSWZ256rr $ymm0, $ymm1 + $ymm0 = VPMULHRSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMULHUWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMULHUWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMULHUWYrr $ymm0, $ymm1 - $ymm0 = VPMULHUWZ256rr $ymm0, $ymm1 + $ymm0 = VPMULHUWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMULHWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMULHWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMULHWYrr $ymm0, $ymm1 - $ymm0 = VPMULHWZ256rr $ymm0, $ymm1 + $ymm0 = VPMULHWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMULLDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMULLDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMULLDYrr $ymm0, $ymm1 - $ymm0 = VPMULLDZ256rr $ymm0, $ymm1 + $ymm0 = VPMULLDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMULLWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMULLWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPMULLWYrr $ymm0, $ymm1 - $ymm0 = VPMULLWZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPMULLWYrr $ymm0, $ymm1 + $ymm0 = VPMULLWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMULUDQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMULUDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMULUDQYrr $ymm0, $ymm1 - $ymm0 = VPMULUDQZ256rr $ymm0, $ymm1 + $ymm0 = VPMULUDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPORYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPORDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPORYrr $ymm0, $ymm1 - $ymm0 = VPORDZ256rr $ymm0, $ymm1 + $ymm0 = VPORDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPORYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPORQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPORYrr $ymm0, $ymm1 - $ymm0 = VPORQZ256rr $ymm0, $ymm1 + $ymm0 = VPORQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBBYrr $ymm0, $ymm1 - $ymm0 = VPSUBBZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBDYrr $ymm0, $ymm1 - $ymm0 = VPSUBDZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBQYrr $ymm0, $ymm1 - $ymm0 = VPSUBQZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBSBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBSBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBSBYrr $ymm0, $ymm1 - $ymm0 = VPSUBSBZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBSBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBSWYrr $ymm0, $ymm1 - $ymm0 = VPSUBSWZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBUSBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBUSBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBUSBYrr $ymm0, $ymm1 - $ymm0 = VPSUBUSBZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBUSBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBUSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBUSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBUSWYrr $ymm0, $ymm1 - $ymm0 = VPSUBUSWZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBUSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBWYrr $ymm0, $ymm1 - $ymm0 = VPSUBWZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPXORYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPXORDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPXORYrr $ymm0, $ymm1 - $ymm0 = VPXORDZ256rr $ymm0, $ymm1 + $ymm0 = VPXORDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPXORYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPXORQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPXORYrr $ymm0, $ymm1 - $ymm0 = VPXORQZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPXORYrr $ymm0, $ymm1 + $ymm0 = VPXORQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VADDPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm0 = VADDPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm0 = VADDPDYrr $ymm0, $ymm1, implicit $mxcsr @@ -308,19 +308,19 @@ body: | ; CHECK: $ymm0 = VANDNPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VANDNPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VANDNPDYrr $ymm0, $ymm1 - $ymm0 = VANDNPDZ256rr $ymm0, $ymm1 + $ymm0 = VANDNPDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VANDNPSYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VANDNPSZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VANDNPSYrr $ymm0, $ymm1 - $ymm0 = VANDNPSZ256rr $ymm0, $ymm1 + $ymm0 = VANDNPSZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VANDPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VANDPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VANDPDYrr $ymm0, $ymm1 - $ymm0 = VANDPDZ256rr $ymm0, $ymm1 + $ymm0 = VANDPDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VANDPSYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VANDPSZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VANDPSYrr $ymm0, $ymm1 - $ymm0 = VANDPSZ256rr $ymm0, $ymm1 + $ymm0 = VANDPSZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VDIVPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm0 = VDIVPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm0 = VDIVPDYrr $ymm0, $ymm1, implicit $mxcsr @@ -364,43 +364,43 @@ body: | ; CHECK: $ymm0 = VXORPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VXORPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VXORPDYrr $ymm0, $ymm1 - $ymm0 = VXORPDZ256rr $ymm0, $ymm1 + $ymm0 = VXORPDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VXORPSYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VXORPSZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VXORPSYrr $ymm0, $ymm1 - $ymm0 = VXORPSZ256rr $ymm0, $ymm1 + $ymm0 = VXORPSZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPACKSSDWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPACKSSDWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPACKSSDWYrr $ymm0, $ymm1 - $ymm0 = VPACKSSDWZ256rr $ymm0, $ymm1 + $ymm0 = VPACKSSDWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPACKSSWBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPACKSSWBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPACKSSWBYrr $ymm0, $ymm1 - $ymm0 = VPACKSSWBZ256rr $ymm0, $ymm1 + $ymm0 = VPACKSSWBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPACKUSDWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPACKUSDWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPACKUSDWYrr $ymm0, $ymm1 - $ymm0 = VPACKUSDWZ256rr $ymm0, $ymm1 + $ymm0 = VPACKUSDWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPACKUSWBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPACKUSWBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPACKUSWBYrr $ymm0, $ymm1 - $ymm0 = VPACKUSWBZ256rr $ymm0, $ymm1 + $ymm0 = VPACKUSWBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VUNPCKHPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VUNPCKHPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VUNPCKHPDYrr $ymm0, $ymm1 - $ymm0 = VUNPCKHPDZ256rr $ymm0, $ymm1 + $ymm0 = VUNPCKHPDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VUNPCKHPSYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VUNPCKHPSZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VUNPCKHPSYrr $ymm0, $ymm1 - $ymm0 = VUNPCKHPSZ256rr $ymm0, $ymm1 + $ymm0 = VUNPCKHPSZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VUNPCKLPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VUNPCKLPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VUNPCKLPDYrr $ymm0, $ymm1 - $ymm0 = VUNPCKLPDZ256rr $ymm0, $ymm1 + $ymm0 = VUNPCKLPDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VUNPCKLPSYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VUNPCKLPSZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VUNPCKLPSYrr $ymm0, $ymm1 - $ymm0 = VUNPCKLPSZ256rr $ymm0, $ymm1 + $ymm0 = VUNPCKLPSZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VSUBPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm0 = VSUBPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm0 = VSUBPDYrr $ymm0, $ymm1, implicit $mxcsr @@ -412,35 +412,35 @@ body: | ; CHECK: $ymm0 = VPUNPCKHBWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKHBWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPUNPCKHBWYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKHBWZ256rr $ymm0, $ymm1 + $ymm0 = VPUNPCKHBWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPUNPCKHDQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKHDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPUNPCKHDQYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKHDQZ256rr $ymm0, $ymm1 + $ymm0 = VPUNPCKHDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPUNPCKHQDQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKHQDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPUNPCKHQDQYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKHQDQZ256rr $ymm0, $ymm1 + $ymm0 = VPUNPCKHQDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPUNPCKHWDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKHWDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPUNPCKHWDYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKHWDZ256rr $ymm0, $ymm1 + $ymm0 = VPUNPCKHWDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPUNPCKLBWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKLBWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPUNPCKLBWYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKLBWZ256rr $ymm0, $ymm1 + $ymm0 = VPUNPCKLBWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPUNPCKLDQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKLDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPUNPCKLDQYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKLDQZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPUNPCKLDQYrr $ymm0, $ymm1 + $ymm0 = VPUNPCKLDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPUNPCKLQDQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKLQDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPUNPCKLQDQYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKLQDQZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPUNPCKLQDQYrr $ymm0, $ymm1 + $ymm0 = VPUNPCKLQDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPUNPCKLWDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKLWDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPUNPCKLWDYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKLWDZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPUNPCKLWDYrr $ymm0, $ymm1 + $ymm0 = VPUNPCKLWDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VFMADD132PDYm $ymm0, $ymm0, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm0 = VFMADD132PDZ256m $ymm0, $ymm0, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm0 = VFMADD132PDYr $ymm0, $ymm1, $ymm2, implicit $mxcsr @@ -586,98 +586,98 @@ body: | ; CHECK: $ymm0 = VFNMSUB231PSYr $ymm0, $ymm1, $ymm2, implicit $mxcsr $ymm0 = VFNMSUB231PSZ256r $ymm0, $ymm1, $ymm2, implicit $mxcsr ; CHECK: $ymm0 = VPSRADYri $ymm0, 7 - $ymm0 = VPSRADZ256ri $ymm0, 7 + $ymm0 = VPSRADZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSRADYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRADZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSRADYrr $ymm0, $xmm1 - $ymm0 = VPSRADZ256rr $ymm0, $xmm1 + $ymm0 = VPSRADZ256rr $ymm0, $xmm1 ; CHECK: $ymm0 = VPSRAVDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRAVDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSRAVDYrr $ymm0, $ymm1 - $ymm0 = VPSRAVDZ256rr $ymm0, $ymm1 + $ymm0 = VPSRAVDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSRAWYri $ymm0, 7 - $ymm0 = VPSRAWZ256ri $ymm0, 7 + $ymm0 = VPSRAWZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSRAWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRAWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSRAWYrr $ymm0, $xmm1 - $ymm0 = VPSRAWZ256rr $ymm0, $xmm1 + $ymm0 = VPSRAWZ256rr $ymm0, $xmm1 ; CHECK: $ymm0 = VPSRLDQYri $ymm0, 7 $ymm0 = VPSRLDQZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSRLDYri $ymm0, 7 - $ymm0 = VPSRLDZ256ri $ymm0, 7 + $ymm0 = VPSRLDZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSRLDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRLDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSRLDYrr $ymm0, $xmm1 - $ymm0 = VPSRLDZ256rr $ymm0, $xmm1 + $ymm0 = VPSRLDZ256rr $ymm0, $xmm1 ; CHECK: $ymm0 = VPSRLQYri $ymm0, 7 - $ymm0 = VPSRLQZ256ri $ymm0, 7 + $ymm0 = VPSRLQZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSRLQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRLQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSRLQYrr $ymm0, $xmm1 - $ymm0 = VPSRLQZ256rr $ymm0, $xmm1 + $ymm0 = VPSRLQZ256rr $ymm0, $xmm1 ; CHECK: $ymm0 = VPSRLVDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRLVDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSRLVDYrr $ymm0, $ymm1 - $ymm0 = VPSRLVDZ256rr $ymm0, $ymm1 + $ymm0 = VPSRLVDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSRLVQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRLVQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSRLVQYrr $ymm0, $ymm1 - $ymm0 = VPSRLVQZ256rr $ymm0, $ymm1 + $ymm0 = VPSRLVQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSRLWYri $ymm0, 7 - $ymm0 = VPSRLWZ256ri $ymm0, 7 + $ymm0 = VPSRLWZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSRLWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRLWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPSRLWYrr $ymm0, $xmm1 - $ymm0 = VPSRLWZ256rr $ymm0, $xmm1 + ; CHECK: $ymm0 = VPSRLWYrr $ymm0, $xmm1 + $ymm0 = VPSRLWZ256rr $ymm0, $xmm1 ; CHECK: $ymm0 = VPMOVSXBDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVSXBDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVSXBDYrr $xmm0 - $ymm0 = VPMOVSXBDZ256rr $xmm0 + $ymm0 = VPMOVSXBDZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVSXBQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVSXBQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVSXBQYrr $xmm0 - $ymm0 = VPMOVSXBQZ256rr $xmm0 + $ymm0 = VPMOVSXBQZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVSXBWYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVSXBWZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVSXBWYrr $xmm0 - $ymm0 = VPMOVSXBWZ256rr $xmm0 + $ymm0 = VPMOVSXBWZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVSXDQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVSXDQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVSXDQYrr $xmm0 - $ymm0 = VPMOVSXDQZ256rr $xmm0 + $ymm0 = VPMOVSXDQZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVSXWDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVSXWDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVSXWDYrr $xmm0 - $ymm0 = VPMOVSXWDZ256rr $xmm0 + $ymm0 = VPMOVSXWDZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVSXWQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVSXWQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVSXWQYrr $xmm0 - $ymm0 = VPMOVSXWQZ256rr $xmm0 + $ymm0 = VPMOVSXWQZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVZXBDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVZXBDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVZXBDYrr $xmm0 - $ymm0 = VPMOVZXBDZ256rr $xmm0 + $ymm0 = VPMOVZXBDZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVZXBQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVZXBQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVZXBQYrr $xmm0 - $ymm0 = VPMOVZXBQZ256rr $xmm0 + $ymm0 = VPMOVZXBQZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVZXBWYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVZXBWZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVZXBWYrr $xmm0 - $ymm0 = VPMOVZXBWZ256rr $xmm0 + $ymm0 = VPMOVZXBWZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVZXDQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVZXDQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVZXDQYrr $xmm0 - $ymm0 = VPMOVZXDQZ256rr $xmm0 + $ymm0 = VPMOVZXDQZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVZXWDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVZXWDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVZXWDYrr $xmm0 - $ymm0 = VPMOVZXWDZ256rr $xmm0 + $ymm0 = VPMOVZXWDZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVZXWQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVZXWQZ256rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPMOVZXWQYrr $xmm0 - $ymm0 = VPMOVZXWQZ256rr $xmm0 - ; CHECK: $ymm0 = VBROADCASTF128 $rip, 1, $noreg, 0, $noreg + ; CHECK: $ymm0 = VPMOVZXWQYrr $xmm0 + $ymm0 = VPMOVZXWQZ256rr $xmm0 + ; CHECK: $ymm0 = VBROADCASTF128rm $rip, 1, $noreg, 0, $noreg $ymm0 = VBROADCASTF32X4Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VBROADCASTSDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VBROADCASTF32X2Z256rm $rip, 1, $noreg, 0, $noreg @@ -686,24 +686,24 @@ body: | ; CHECK: $ymm0 = VBROADCASTSDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VBROADCASTSDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VBROADCASTSDYrr $xmm0 - $ymm0 = VBROADCASTSDZ256rr $xmm0 + $ymm0 = VBROADCASTSDZ256rr $xmm0 ; CHECK: $ymm0 = VBROADCASTSSYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VBROADCASTSSZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VBROADCASTSSYrr $xmm0 - $ymm0 = VBROADCASTSSZ256rr $xmm0 + $ymm0 = VBROADCASTSSZ256rr $xmm0 ; CHECK: $ymm0 = VPBROADCASTBYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPBROADCASTBZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPBROADCASTBYrr $xmm0 - $ymm0 = VPBROADCASTBZ256rr $xmm0 + $ymm0 = VPBROADCASTBZ256rr $xmm0 ; CHECK: $ymm0 = VPBROADCASTDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPBROADCASTDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPBROADCASTDYrr $xmm0 - $ymm0 = VPBROADCASTDZ256rr $xmm0 + $ymm0 = VPBROADCASTDZ256rr $xmm0 ; CHECK: $ymm0 = VPBROADCASTWYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPBROADCASTWZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPBROADCASTWYrr $xmm0 - $ymm0 = VPBROADCASTWZ256rr $xmm0 - ; CHECK: $ymm0 = VBROADCASTI128 $rip, 1, $noreg, 0, $noreg + $ymm0 = VPBROADCASTWZ256rr $xmm0 + ; CHECK: $ymm0 = VBROADCASTI128rm $rip, 1, $noreg, 0, $noreg $ymm0 = VBROADCASTI32X4Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPBROADCASTQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VBROADCASTI32X2Z256rm $rip, 1, $noreg, 0, $noreg @@ -711,66 +711,66 @@ body: | $ymm0 = VBROADCASTI32X2Z256rr $xmm0 ; CHECK: $ymm0 = VPBROADCASTQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPBROADCASTQZ256rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPBROADCASTQYrr $xmm0 - $ymm0 = VPBROADCASTQZ256rr $xmm0 + ; CHECK: $ymm0 = VPBROADCASTQYrr $xmm0 + $ymm0 = VPBROADCASTQZ256rr $xmm0 ; CHECK: $ymm0 = VPABSBYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPABSBZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPABSBYrr $ymm0 - $ymm0 = VPABSBZ256rr $ymm0 + $ymm0 = VPABSBZ256rr $ymm0 ; CHECK: $ymm0 = VPABSDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPABSDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPABSDYrr $ymm0 - $ymm0 = VPABSDZ256rr $ymm0 + $ymm0 = VPABSDZ256rr $ymm0 ; CHECK: $ymm0 = VPABSWYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPABSWZ256rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPABSWYrr $ymm0 - $ymm0 = VPABSWZ256rr $ymm0 + ; CHECK: $ymm0 = VPABSWYrr $ymm0 + $ymm0 = VPABSWZ256rr $ymm0 ; CHECK: $ymm0 = VPSADBWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSADBWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPSADBWYrr $ymm0, $ymm1 - $ymm0 = VPSADBWZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPSADBWYrr $ymm0, $ymm1 + $ymm0 = VPSADBWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPERMDYrm $ymm0, $rdi, 1, $noreg, 0, $noreg - $ymm0 = VPERMDZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg + $ymm0 = VPERMDZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPERMDYrr $ymm1, $ymm0 - $ymm0 = VPERMDZ256rr $ymm1, $ymm0 + $ymm0 = VPERMDZ256rr $ymm1, $ymm0 ; CHECK: $ymm0 = VPERMILPDYmi $rdi, 1, $noreg, 0, $noreg, 7 $ymm0 = VPERMILPDZ256mi $rdi, 1, $noreg, 0, $noreg, 7 ; CHECK: $ymm0 = VPERMILPDYri $ymm0, 7 - $ymm0 = VPERMILPDZ256ri $ymm0, 7 + $ymm0 = VPERMILPDZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPERMILPDYrm $ymm0, $rdi, 1, $noreg, 0, $noreg - $ymm0 = VPERMILPDZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg + $ymm0 = VPERMILPDZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPERMILPDYrr $ymm1, $ymm0 - $ymm0 = VPERMILPDZ256rr $ymm1, $ymm0 + $ymm0 = VPERMILPDZ256rr $ymm1, $ymm0 ; CHECK: $ymm0 = VPERMILPSYmi $rdi, 1, $noreg, 0, $noreg, 7 $ymm0 = VPERMILPSZ256mi $rdi, 1, $noreg, 0, $noreg, 7 ; CHECK: $ymm0 = VPERMILPSYri $ymm0, 7 - $ymm0 = VPERMILPSZ256ri $ymm0, 7 + $ymm0 = VPERMILPSZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPERMILPSYrm $ymm0, $rdi, 1, $noreg, 0, $noreg - $ymm0 = VPERMILPSZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg + $ymm0 = VPERMILPSZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPERMILPSYrr $ymm1, $ymm0 - $ymm0 = VPERMILPSZ256rr $ymm1, $ymm0 + $ymm0 = VPERMILPSZ256rr $ymm1, $ymm0 ; CHECK: $ymm0 = VPERMPDYmi $rdi, 1, $noreg, 0, $noreg, 7 $ymm0 = VPERMPDZ256mi $rdi, 1, $noreg, 0, $noreg, 7 ; CHECK: $ymm0 = VPERMPDYri $ymm0, 7 - $ymm0 = VPERMPDZ256ri $ymm0, 7 + $ymm0 = VPERMPDZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPERMPSYrm $ymm0, $rdi, 1, $noreg, 0, $noreg - $ymm0 = VPERMPSZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg + $ymm0 = VPERMPSZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPERMPSYrr $ymm1, $ymm0 - $ymm0 = VPERMPSZ256rr $ymm1, $ymm0 + $ymm0 = VPERMPSZ256rr $ymm1, $ymm0 ; CHECK: $ymm0 = VPERMQYmi $rdi, 1, $noreg, 0, $noreg, 7 $ymm0 = VPERMQZ256mi $rdi, 1, $noreg, 0, $noreg, 7 - ; CHECK: $ymm0 = VPERMQYri $ymm0, 7 - $ymm0 = VPERMQZ256ri $ymm0, 7 + ; CHECK: $ymm0 = VPERMQYri $ymm0, 7 + $ymm0 = VPERMQZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSLLDQYri $ymm0, 14 - $ymm0 = VPSLLDQZ256ri $ymm0, 14 + $ymm0 = VPSLLDQZ256ri $ymm0, 14 ; CHECK: $ymm0 = VPSLLDYri $ymm0, 7 - $ymm0 = VPSLLDZ256ri $ymm0, 7 + $ymm0 = VPSLLDZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSLLDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSLLDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSLLDYrr $ymm0, $xmm0 $ymm0 = VPSLLDZ256rr $ymm0, $xmm0 ; CHECK: $ymm0 = VPSLLQYri $ymm0, 7 - $ymm0 = VPSLLQZ256ri $ymm0, 7 + $ymm0 = VPSLLQZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSLLQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSLLQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSLLQYrr $ymm0, $xmm0 @@ -784,7 +784,7 @@ body: | ; CHECK: $ymm0 = VPSLLVQYrr $ymm0, $ymm0 $ymm0 = VPSLLVQZ256rr $ymm0, $ymm0 ; CHECK: $ymm0 = VPSLLWYri $ymm0, 7 - $ymm0 = VPSLLWZ256ri $ymm0, 7 + $ymm0 = VPSLLWZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSLLWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSLLWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSLLWYrr $ymm0, $xmm0 @@ -792,7 +792,7 @@ body: | ; CHECK: $ymm0 = VCVTDQ2PDYrm $rdi, 1, $noreg, 0, $noreg $ymm0 = VCVTDQ2PDZ256rm $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VCVTDQ2PDYrr $xmm0 - $ymm0 = VCVTDQ2PDZ256rr $xmm0 + $ymm0 = VCVTDQ2PDZ256rr $xmm0 ; CHECK: $ymm0 = VCVTDQ2PSYrm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm0 = VCVTDQ2PSZ256rm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm0 = VCVTDQ2PSYrr $ymm0, implicit $mxcsr @@ -841,26 +841,26 @@ body: | $ymm0 = VPALIGNRZ256rmi $ymm0, $rdi, 1, $noreg, 0, $noreg, 1 ; CHECK: $ymm0 = VPALIGNRYrri $ymm0, $ymm1, 1 $ymm0 = VPALIGNRZ256rri $ymm0, $ymm1, 1 - ; CHECK: $ymm0 = VMOVUPSYrm $rdi, 1, $noreg, 0, $noreg - $ymm0 = VMOVUPSZ256rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $ymm0 = VMOVUPSYrm $rdi, 1, $noreg, 0, $noreg + $ymm0 = VMOVUPSZ256rm $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVUPSYrr $ymm0 - $ymm0 = VMOVUPSZ256rr $ymm0 + $ymm0 = VMOVUPSZ256rr $ymm0 ; CHECK: $ymm0 = VPSHUFBYrm $ymm0, $rdi, 1, $noreg, 0, $noreg $ymm0 = VPSHUFBZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSHUFBYrr $ymm0, $ymm1 - $ymm0 = VPSHUFBZ256rr $ymm0, $ymm1 + $ymm0 = VPSHUFBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSHUFDYmi $rdi, 1, $noreg, 0, $noreg, -24 $ymm0 = VPSHUFDZ256mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm0 = VPSHUFDYri $ymm0, -24 - $ymm0 = VPSHUFDZ256ri $ymm0, -24 + $ymm0 = VPSHUFDZ256ri $ymm0, -24 ; CHECK: $ymm0 = VPSHUFHWYmi $rdi, 1, $noreg, 0, $noreg, -24 $ymm0 = VPSHUFHWZ256mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm0 = VPSHUFHWYri $ymm0, -24 - $ymm0 = VPSHUFHWZ256ri $ymm0, -24 + $ymm0 = VPSHUFHWZ256ri $ymm0, -24 ; CHECK: $ymm0 = VPSHUFLWYmi $rdi, 1, $noreg, 0, $noreg, -24 $ymm0 = VPSHUFLWZ256mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm0 = VPSHUFLWYri $ymm0, -24 - $ymm0 = VPSHUFLWZ256ri $ymm0, -24 + $ymm0 = VPSHUFLWZ256ri $ymm0, -24 ; CHECK: $ymm0 = VSHUFPDYrmi $ymm0, $rdi, 1, $noreg, 0, $noreg, -24 $ymm0 = VSHUFPDZ256rmi $ymm0, $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm0 = VSHUFPDYrri $ymm0, $ymm1, -24 @@ -904,137 +904,137 @@ name: evex_z128_to_vex_test body: | bb.0: ; CHECK: VMOVAPDmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVAPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVAPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVAPDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVAPDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVAPDrr $xmm0 - $xmm0 = VMOVAPDZ128rr $xmm0 + $xmm0 = VMOVAPDZ128rr $xmm0 ; CHECK: VMOVAPSmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVAPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVAPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVAPSrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVAPSZ128rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VMOVAPSrr $xmm0 - $xmm0 = VMOVAPSZ128rr $xmm0 + ; CHECK: $xmm0 = VMOVAPSrr $xmm0 + $xmm0 = VMOVAPSZ128rr $xmm0 ; CHECK: VMOVDQAmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVDQA32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVDQA32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVDQArm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVDQA32Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVDQArr $xmm0 - $xmm0 = VMOVDQA32Z128rr $xmm0 + $xmm0 = VMOVDQA32Z128rr $xmm0 ; CHECK: VMOVDQAmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVDQA64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVDQA64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVDQArm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVDQA64Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVDQArr $xmm0 - $xmm0 = VMOVDQA64Z128rr $xmm0 + $xmm0 = VMOVDQA64Z128rr $xmm0 ; CHECK: VMOVDQUmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVDQU16Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVDQU16Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVDQUrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVDQU16Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVDQUrr $xmm0 - $xmm0 = VMOVDQU16Z128rr $xmm0 + $xmm0 = VMOVDQU16Z128rr $xmm0 ; CHECK: VMOVDQUmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVDQU32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVDQU32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVDQUrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVDQU32Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVDQUrr $xmm0 - $xmm0 = VMOVDQU32Z128rr $xmm0 + $xmm0 = VMOVDQU32Z128rr $xmm0 ; CHECK: VMOVDQUmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVDQU64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVDQU64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVDQUrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVDQU64Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVDQUrr $xmm0 - $xmm0 = VMOVDQU64Z128rr $xmm0 + $xmm0 = VMOVDQU64Z128rr $xmm0 ; CHECK: VMOVDQUmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVDQU8Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVDQU8Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVDQUrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVDQU8Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVDQUrr $xmm0 - $xmm0 = VMOVDQU8Z128rr $xmm0 + $xmm0 = VMOVDQU8Z128rr $xmm0 ; CHECK: $xmm0 = VMOVNTDQArm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVNTDQAZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: VMOVUPDmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVUPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVUPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVUPDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVUPDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVUPDrr $xmm0 - $xmm0 = VMOVUPDZ128rr $xmm0 + $xmm0 = VMOVUPDZ128rr $xmm0 ; CHECK: VMOVUPSmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVUPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVUPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVUPSrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVUPSZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVUPSrr $xmm0 - $xmm0 = VMOVUPSZ128rr $xmm0 + $xmm0 = VMOVUPSZ128rr $xmm0 ; CHECK: VMOVNTDQmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVNTDQZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVNTDQZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: VMOVNTPDmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVNTPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVNTPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: VMOVNTPSmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVNTPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVNTPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VPMOVSXBDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVSXBDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVSXBDrr $xmm0 - $xmm0 = VPMOVSXBDZ128rr $xmm0 + $xmm0 = VPMOVSXBDZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVSXBQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVSXBQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVSXBQrr $xmm0 - $xmm0 = VPMOVSXBQZ128rr $xmm0 + $xmm0 = VPMOVSXBQZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVSXBWrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVSXBWZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVSXBWrr $xmm0 - $xmm0 = VPMOVSXBWZ128rr $xmm0 + $xmm0 = VPMOVSXBWZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVSXDQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVSXDQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVSXDQrr $xmm0 - $xmm0 = VPMOVSXDQZ128rr $xmm0 + $xmm0 = VPMOVSXDQZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVSXWDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVSXWDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVSXWDrr $xmm0 - $xmm0 = VPMOVSXWDZ128rr $xmm0 + $xmm0 = VPMOVSXWDZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVSXWQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVSXWQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVSXWQrr $xmm0 - $xmm0 = VPMOVSXWQZ128rr $xmm0 + $xmm0 = VPMOVSXWQZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVZXBDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVZXBDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVZXBDrr $xmm0 - $xmm0 = VPMOVZXBDZ128rr $xmm0 + $xmm0 = VPMOVZXBDZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVZXBQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVZXBQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVZXBQrr $xmm0 - $xmm0 = VPMOVZXBQZ128rr $xmm0 + $xmm0 = VPMOVZXBQZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVZXBWrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVZXBWZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVZXBWrr $xmm0 - $xmm0 = VPMOVZXBWZ128rr $xmm0 + $xmm0 = VPMOVZXBWZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVZXDQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVZXDQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVZXDQrr $xmm0 - $xmm0 = VPMOVZXDQZ128rr $xmm0 + $xmm0 = VPMOVZXDQZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVZXWDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVZXWDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVZXWDrr $xmm0 - $xmm0 = VPMOVZXWDZ128rr $xmm0 + $xmm0 = VPMOVZXWDZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVZXWQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVZXWQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVZXWQrr $xmm0 - $xmm0 = VPMOVZXWQZ128rr $xmm0 + $xmm0 = VPMOVZXWQZ128rr $xmm0 ; CHECK: VMOVHPDmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVHPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVHPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVHPDrm $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VMOVHPDZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg + $xmm0 = VMOVHPDZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: VMOVHPSmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVHPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVHPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVHPSrm $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VMOVHPSZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg + $xmm0 = VMOVHPSZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: VMOVLPDmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVLPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVLPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVLPDrm $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VMOVLPDZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg + $xmm0 = VMOVLPDZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: VMOVLPSmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVLPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 - ; CHECK: $xmm0 = VMOVLPSrm $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VMOVLPSZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg + VMOVLPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + ; CHECK: $xmm0 = VMOVLPSrm $xmm0, $rdi, 1, $noreg, 0, $noreg + $xmm0 = VMOVLPSZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMAXCPDrm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VMAXCPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VMAXCPDrr $xmm0, $xmm1, implicit $mxcsr @@ -1078,183 +1078,183 @@ body: | ; CHECK: $xmm0 = VORPDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VORPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VORPDrr $xmm0, $xmm1 - $xmm0 = VORPDZ128rr $xmm0, $xmm1 + $xmm0 = VORPDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VORPSrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VORPSZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VORPSrr $xmm0, $xmm1 - $xmm0 = VORPSZ128rr $xmm0, $xmm1 + $xmm0 = VORPSZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDBrr $xmm0, $xmm1 - $xmm0 = VPADDBZ128rr $xmm0, $xmm1 + $xmm0 = VPADDBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDDrr $xmm0, $xmm1 - $xmm0 = VPADDDZ128rr $xmm0, $xmm1 + $xmm0 = VPADDDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDQrr $xmm0, $xmm1 - $xmm0 = VPADDQZ128rr $xmm0, $xmm1 + $xmm0 = VPADDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDSBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDSBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDSBrr $xmm0, $xmm1 - $xmm0 = VPADDSBZ128rr $xmm0, $xmm1 + $xmm0 = VPADDSBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDSWrr $xmm0, $xmm1 - $xmm0 = VPADDSWZ128rr $xmm0, $xmm1 + $xmm0 = VPADDSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDUSBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDUSBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDUSBrr $xmm0, $xmm1 - $xmm0 = VPADDUSBZ128rr $xmm0, $xmm1 + $xmm0 = VPADDUSBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDUSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDUSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDUSWrr $xmm0, $xmm1 - $xmm0 = VPADDUSWZ128rr $xmm0, $xmm1 + $xmm0 = VPADDUSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDWrr $xmm0, $xmm1 - $xmm0 = VPADDWZ128rr $xmm0, $xmm1 + $xmm0 = VPADDWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPANDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPANDDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPANDrr $xmm0, $xmm1 - $xmm0 = VPANDDZ128rr $xmm0, $xmm1 + $xmm0 = VPANDDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPANDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPANDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPANDrr $xmm0, $xmm1 - $xmm0 = VPANDQZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPANDrr $xmm0, $xmm1 + $xmm0 = VPANDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPANDNrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPANDNDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPANDNrr $xmm0, $xmm1 - $xmm0 = VPANDNDZ128rr $xmm0, $xmm1 + $xmm0 = VPANDNDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPANDNrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPANDNQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPANDNrr $xmm0, $xmm1 - $xmm0 = VPANDNQZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPANDNrr $xmm0, $xmm1 + $xmm0 = VPANDNQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPAVGBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPAVGBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPAVGBrr $xmm0, $xmm1 - $xmm0 = VPAVGBZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPAVGBrr $xmm0, $xmm1 + $xmm0 = VPAVGBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPAVGWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPAVGWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPAVGWrr $xmm0, $xmm1 - $xmm0 = VPAVGWZ128rr $xmm0, $xmm1 + $xmm0 = VPAVGWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMAXSBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMAXSBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMAXSBrr $xmm0, $xmm1 - $xmm0 = VPMAXSBZ128rr $xmm0, $xmm1 + $xmm0 = VPMAXSBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMAXSDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMAXSDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMAXSDrr $xmm0, $xmm1 - $xmm0 = VPMAXSDZ128rr $xmm0, $xmm1 + $xmm0 = VPMAXSDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMAXSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMAXSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPMAXSWrr $xmm0, $xmm1 - $xmm0 = VPMAXSWZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPMAXSWrr $xmm0, $xmm1 + $xmm0 = VPMAXSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMAXUBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMAXUBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMAXUBrr $xmm0, $xmm1 - $xmm0 = VPMAXUBZ128rr $xmm0, $xmm1 + $xmm0 = VPMAXUBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMAXUDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMAXUDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMAXUDrr $xmm0, $xmm1 - $xmm0 = VPMAXUDZ128rr $xmm0, $xmm1 + $xmm0 = VPMAXUDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMAXUWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMAXUWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMAXUWrr $xmm0, $xmm1 - $xmm0 = VPMAXUWZ128rr $xmm0, $xmm1 + $xmm0 = VPMAXUWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMINSBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMINSBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMINSBrr $xmm0, $xmm1 - $xmm0 = VPMINSBZ128rr $xmm0, $xmm1 + $xmm0 = VPMINSBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMINSDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMINSDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMINSDrr $xmm0, $xmm1 - $xmm0 = VPMINSDZ128rr $xmm0, $xmm1 + $xmm0 = VPMINSDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMINSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMINSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMINSWrr $xmm0, $xmm1 - $xmm0 = VPMINSWZ128rr $xmm0, $xmm1 + $xmm0 = VPMINSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMINUBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMINUBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMINUBrr $xmm0, $xmm1 - $xmm0 = VPMINUBZ128rr $xmm0, $xmm1 + $xmm0 = VPMINUBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMINUDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMINUDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMINUDrr $xmm0, $xmm1 - $xmm0 = VPMINUDZ128rr $xmm0, $xmm1 + $xmm0 = VPMINUDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMINUWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMINUWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMINUWrr $xmm0, $xmm1 - $xmm0 = VPMINUWZ128rr $xmm0, $xmm1 + $xmm0 = VPMINUWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMULDQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMULDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMULDQrr $xmm0, $xmm1 - $xmm0 = VPMULDQZ128rr $xmm0, $xmm1 + $xmm0 = VPMULDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMULHRSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMULHRSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMULHRSWrr $xmm0, $xmm1 - $xmm0 = VPMULHRSWZ128rr $xmm0, $xmm1 + $xmm0 = VPMULHRSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMULHUWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMULHUWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMULHUWrr $xmm0, $xmm1 - $xmm0 = VPMULHUWZ128rr $xmm0, $xmm1 + $xmm0 = VPMULHUWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMULHWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMULHWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMULHWrr $xmm0, $xmm1 - $xmm0 = VPMULHWZ128rr $xmm0, $xmm1 + $xmm0 = VPMULHWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMULLDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMULLDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMULLDrr $xmm0, $xmm1 - $xmm0 = VPMULLDZ128rr $xmm0, $xmm1 + $xmm0 = VPMULLDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMULLWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMULLWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMULLWrr $xmm0, $xmm1 - $xmm0 = VPMULLWZ128rr $xmm0, $xmm1 + $xmm0 = VPMULLWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMULUDQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMULUDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMULUDQrr $xmm0, $xmm1 - $xmm0 = VPMULUDQZ128rr $xmm0, $xmm1 + $xmm0 = VPMULUDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPORrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPORDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPORrr $xmm0, $xmm1 - $xmm0 = VPORDZ128rr $xmm0, $xmm1 + $xmm0 = VPORDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPORrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPORQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPORrr $xmm0, $xmm1 - $xmm0 = VPORQZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPORrr $xmm0, $xmm1 + $xmm0 = VPORQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSUBBrr $xmm0, $xmm1 - $xmm0 = VPSUBBZ128rr $xmm0, $xmm1 + $xmm0 = VPSUBBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSUBDrr $xmm0, $xmm1 - $xmm0 = VPSUBDZ128rr $xmm0, $xmm1 + $xmm0 = VPSUBDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSUBQrr $xmm0, $xmm1 - $xmm0 = VPSUBQZ128rr $xmm0, $xmm1 + $xmm0 = VPSUBQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBSBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBSBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPSUBSBrr $xmm0, $xmm1 - $xmm0 = VPSUBSBZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPSUBSBrr $xmm0, $xmm1 + $xmm0 = VPSUBSBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSUBSWrr $xmm0, $xmm1 - $xmm0 = VPSUBSWZ128rr $xmm0, $xmm1 + $xmm0 = VPSUBSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBUSBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBUSBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPSUBUSBrr $xmm0, $xmm1 - $xmm0 = VPSUBUSBZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPSUBUSBrr $xmm0, $xmm1 + $xmm0 = VPSUBUSBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBUSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBUSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSUBUSWrr $xmm0, $xmm1 - $xmm0 = VPSUBUSWZ128rr $xmm0, $xmm1 + $xmm0 = VPSUBUSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPSUBWrr $xmm0, $xmm1 - $xmm0 = VPSUBWZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPSUBWrr $xmm0, $xmm1 + $xmm0 = VPSUBWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VADDPDrm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VADDPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VADDPDrr $xmm0, $xmm1, implicit $mxcsr @@ -1266,19 +1266,19 @@ body: | ; CHECK: $xmm0 = VANDNPDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VANDNPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VANDNPDrr $xmm0, $xmm1 - $xmm0 = VANDNPDZ128rr $xmm0, $xmm1 + $xmm0 = VANDNPDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VANDNPSrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VANDNPSZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VANDNPSrr $xmm0, $xmm1 - $xmm0 = VANDNPSZ128rr $xmm0, $xmm1 + $xmm0 = VANDNPSZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VANDPDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VANDPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VANDPDrr $xmm0, $xmm1 - $xmm0 = VANDPDZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VANDPDrr $xmm0, $xmm1 + $xmm0 = VANDPDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VANDPSrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VANDPSZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VANDPSrr $xmm0, $xmm1 - $xmm0 = VANDPSZ128rr $xmm0, $xmm1 + $xmm0 = VANDPSZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VDIVPDrm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VDIVPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VDIVPDrr $xmm0, $xmm1, implicit $mxcsr @@ -1290,11 +1290,11 @@ body: | ; CHECK: $xmm0 = VPXORrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPXORDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPXORrr $xmm0, $xmm1 - $xmm0 = VPXORDZ128rr $xmm0, $xmm1 + $xmm0 = VPXORDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPXORrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPXORQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPXORrr $xmm0, $xmm1 - $xmm0 = VPXORQZ128rr $xmm0, $xmm1 + $xmm0 = VPXORQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VSUBPDrm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VSUBPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VSUBPDrr $xmm0, $xmm1, implicit $mxcsr @@ -1306,83 +1306,83 @@ body: | ; CHECK: $xmm0 = VXORPDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VXORPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VXORPDrr $xmm0, $xmm1 - $xmm0 = VXORPDZ128rr $xmm0, $xmm1 + $xmm0 = VXORPDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VXORPSrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VXORPSZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VXORPSrr $xmm0, $xmm1 - $xmm0 = VXORPSZ128rr $xmm0, $xmm1 + $xmm0 = VXORPSZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMADDUBSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMADDUBSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMADDUBSWrr $xmm0, $xmm1 - $xmm0 = VPMADDUBSWZ128rr $xmm0, $xmm1 + $xmm0 = VPMADDUBSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMADDWDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMADDWDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPMADDWDrr $xmm0, $xmm1 - $xmm0 = VPMADDWDZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPMADDWDrr $xmm0, $xmm1 + $xmm0 = VPMADDWDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPACKSSDWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPACKSSDWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPACKSSDWrr $xmm0, $xmm1 - $xmm0 = VPACKSSDWZ128rr $xmm0, $xmm1 + $xmm0 = VPACKSSDWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPACKSSWBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPACKSSWBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPACKSSWBrr $xmm0, $xmm1 - $xmm0 = VPACKSSWBZ128rr $xmm0, $xmm1 + $xmm0 = VPACKSSWBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPACKUSDWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPACKUSDWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPACKUSDWrr $xmm0, $xmm1 - $xmm0 = VPACKUSDWZ128rr $xmm0, $xmm1 + $xmm0 = VPACKUSDWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPACKUSWBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPACKUSWBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPACKUSWBrr $xmm0, $xmm1 - $xmm0 = VPACKUSWBZ128rr $xmm0, $xmm1 + $xmm0 = VPACKUSWBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKHBWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKHBWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKHBWrr $xmm0, $xmm1 - $xmm0 = VPUNPCKHBWZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKHBWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKHDQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKHDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKHDQrr $xmm0, $xmm1 - $xmm0 = VPUNPCKHDQZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKHDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKHQDQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKHQDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKHQDQrr $xmm0, $xmm1 - $xmm0 = VPUNPCKHQDQZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKHQDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKHWDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKHWDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKHWDrr $xmm0, $xmm1 - $xmm0 = VPUNPCKHWDZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKHWDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKLBWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKLBWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKLBWrr $xmm0, $xmm1 - $xmm0 = VPUNPCKLBWZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKLBWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKLDQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKLDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKLDQrr $xmm0, $xmm1 - $xmm0 = VPUNPCKLDQZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKLDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKLQDQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKLQDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKLQDQrr $xmm0, $xmm1 - $xmm0 = VPUNPCKLQDQZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKLQDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKLWDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKLWDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKLWDrr $xmm0, $xmm1 - $xmm0 = VPUNPCKLWDZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKLWDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VUNPCKHPDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VUNPCKHPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VUNPCKHPDrr $xmm0, $xmm1 - $xmm0 = VUNPCKHPDZ128rr $xmm0, $xmm1 + $xmm0 = VUNPCKHPDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VUNPCKHPSrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VUNPCKHPSZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VUNPCKHPSrr $xmm0, $xmm1 - $xmm0 = VUNPCKHPSZ128rr $xmm0, $xmm1 + $xmm0 = VUNPCKHPSZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VUNPCKLPDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VUNPCKLPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VUNPCKLPDrr $xmm0, $xmm1 - $xmm0 = VUNPCKLPDZ128rr $xmm0, $xmm1 + $xmm0 = VUNPCKLPDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VUNPCKLPSrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VUNPCKLPSZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VUNPCKLPSrr $xmm0, $xmm1 - $xmm0 = VUNPCKLPSZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VUNPCKLPSrr $xmm0, $xmm1 + $xmm0 = VUNPCKLPSZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VFMADD132PDm $xmm0, $xmm0, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VFMADD132PDZ128m $xmm0, $xmm0, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VFMADD132PDr $xmm0, $xmm1, $xmm2, implicit $mxcsr @@ -1528,13 +1528,13 @@ body: | ; CHECK: $xmm0 = VFNMSUB231PSr $xmm0, $xmm1, $xmm2, implicit $mxcsr $xmm0 = VFNMSUB231PSZ128r $xmm0, $xmm1, $xmm2, implicit $mxcsr ; CHECK: $xmm0 = VPSLLDri $xmm0, 7 - $xmm0 = VPSLLDZ128ri $xmm0, 7 + $xmm0 = VPSLLDZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSLLDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSLLDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSLLDrr $xmm0, $xmm0 $xmm0 = VPSLLDZ128rr $xmm0, $xmm0 ; CHECK: $xmm0 = VPSLLQri $xmm0, 7 - $xmm0 = VPSLLQZ128ri $xmm0, 7 + $xmm0 = VPSLLQZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSLLQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSLLQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSLLQrr $xmm0, $xmm0 @@ -1548,13 +1548,13 @@ body: | ; CHECK: $xmm0 = VPSLLVQrr $xmm0, $xmm0 $xmm0 = VPSLLVQZ128rr $xmm0, $xmm0 ; CHECK: $xmm0 = VPSLLWri $xmm0, 7 - $xmm0 = VPSLLWZ128ri $xmm0, 7 + $xmm0 = VPSLLWZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSLLWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSLLWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSLLWrr $xmm0, $xmm0 $xmm0 = VPSLLWZ128rr $xmm0, $xmm0 ; CHECK: $xmm0 = VPSRADri $xmm0, 7 - $xmm0 = VPSRADZ128ri $xmm0, 7 + $xmm0 = VPSRADZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSRADrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSRADZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSRADrr $xmm0, $xmm0 @@ -1563,22 +1563,22 @@ body: | $xmm0 = VPSRAVDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSRAVDrr $xmm0, $xmm0 $xmm0 = VPSRAVDZ128rr $xmm0, $xmm0 - ; CHECK: $xmm0 = VPSRAWri $xmm0, 7 - $xmm0 = VPSRAWZ128ri $xmm0, 7 + ; CHECK: $xmm0 = VPSRAWri $xmm0, 7 + $xmm0 = VPSRAWZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSRAWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSRAWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSRAWrr $xmm0, $xmm0 $xmm0 = VPSRAWZ128rr $xmm0, $xmm0 ; CHECK: $xmm0 = VPSRLDQri $xmm0, 14 - $xmm0 = VPSRLDQZ128ri $xmm0, 14 - ; CHECK: $xmm0 = VPSRLDri $xmm0, 7 - $xmm0 = VPSRLDZ128ri $xmm0, 7 + $xmm0 = VPSRLDQZ128ri $xmm0, 14 + ; CHECK: $xmm0 = VPSRLDri $xmm0, 7 + $xmm0 = VPSRLDZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSRLDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSRLDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSRLDrr $xmm0, $xmm0 $xmm0 = VPSRLDZ128rr $xmm0, $xmm0 - ; CHECK: $xmm0 = VPSRLQri $xmm0, 7 - $xmm0 = VPSRLQZ128ri $xmm0, 7 + ; CHECK: $xmm0 = VPSRLQri $xmm0, 7 + $xmm0 = VPSRLQZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSRLQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSRLQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSRLQrr $xmm0, $xmm0 @@ -1592,7 +1592,7 @@ body: | ; CHECK: $xmm0 = VPSRLVQrr $xmm0, $xmm0 $xmm0 = VPSRLVQZ128rr $xmm0, $xmm0 ; CHECK: $xmm0 = VPSRLWri $xmm0, 7 - $xmm0 = VPSRLWZ128ri $xmm0, 7 + $xmm0 = VPSRLWZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSRLWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSRLWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSRLWrr $xmm0, $xmm0 @@ -1600,27 +1600,27 @@ body: | ; CHECK: $xmm0 = VPERMILPDmi $rdi, 1, $noreg, 0, $noreg, 9 $xmm0 = VPERMILPDZ128mi $rdi, 1, $noreg, 0, $noreg, 9 ; CHECK: $xmm0 = VPERMILPDri $xmm0, 9 - $xmm0 = VPERMILPDZ128ri $xmm0, 9 + $xmm0 = VPERMILPDZ128ri $xmm0, 9 ; CHECK: $xmm0 = VPERMILPDrm $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VPERMILPDZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg + $xmm0 = VPERMILPDZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPERMILPDrr $xmm0, $xmm1 - $xmm0 = VPERMILPDZ128rr $xmm0, $xmm1 + $xmm0 = VPERMILPDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPERMILPSmi $rdi, 1, $noreg, 0, $noreg, 9 $xmm0 = VPERMILPSZ128mi $rdi, 1, $noreg, 0, $noreg, 9 ; CHECK: $xmm0 = VPERMILPSri $xmm0, 9 - $xmm0 = VPERMILPSZ128ri $xmm0, 9 + $xmm0 = VPERMILPSZ128ri $xmm0, 9 ; CHECK: $xmm0 = VPERMILPSrm $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VPERMILPSZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPERMILPSrr $xmm0, $xmm1 - $xmm0 = VPERMILPSZ128rr $xmm0, $xmm1 + $xmm0 = VPERMILPSZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg + ; CHECK: $xmm0 = VPERMILPSrr $xmm0, $xmm1 + $xmm0 = VPERMILPSZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VCVTPH2PSrm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VCVTPH2PSZ128rm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VCVTPH2PSrr $xmm0, implicit $mxcsr $xmm0 = VCVTPH2PSZ128rr $xmm0, implicit $mxcsr ; CHECK: $xmm0 = VCVTDQ2PDrm $rdi, 1, $noreg, 0, $noreg $xmm0 = VCVTDQ2PDZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VCVTDQ2PDrr $xmm0 - $xmm0 = VCVTDQ2PDZ128rr $xmm0 + ; CHECK: $xmm0 = VCVTDQ2PDrr $xmm0 + $xmm0 = VCVTDQ2PDZ128rr $xmm0 ; CHECK: $xmm0 = VCVTDQ2PSrm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VCVTDQ2PSZ128rm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VCVTDQ2PSrr $xmm0, implicit $mxcsr @@ -1657,34 +1657,34 @@ body: | $xmm0 = VSQRTPSZ128m $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VSQRTPSr $xmm0, implicit $mxcsr $xmm0 = VSQRTPSZ128r $xmm0, implicit $mxcsr - ; CHECK: $xmm0 = VMOVDDUPrm $rdi, 1, $noreg, 0, $noreg - $xmm0 = VMOVDDUPZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VMOVDDUPrr $xmm0 - $xmm0 = VMOVDDUPZ128rr $xmm0 - ; CHECK: $xmm0 = VMOVSHDUPrm $rdi, 1, $noreg, 0, $noreg - $xmm0 = VMOVSHDUPZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VMOVSHDUPrr $xmm0 - $xmm0 = VMOVSHDUPZ128rr $xmm0 - ; CHECK: $xmm0 = VMOVSLDUPrm $rdi, 1, $noreg, 0, $noreg - $xmm0 = VMOVSLDUPZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VMOVSLDUPrr $xmm0 - $xmm0 = VMOVSLDUPZ128rr $xmm0 + ; CHECK: $xmm0 = VMOVDDUPrm $rdi, 1, $noreg, 0, $noreg + $xmm0 = VMOVDDUPZ128rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $xmm0 = VMOVDDUPrr $xmm0 + $xmm0 = VMOVDDUPZ128rr $xmm0 + ; CHECK: $xmm0 = VMOVSHDUPrm $rdi, 1, $noreg, 0, $noreg + $xmm0 = VMOVSHDUPZ128rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $xmm0 = VMOVSHDUPrr $xmm0 + $xmm0 = VMOVSHDUPZ128rr $xmm0 + ; CHECK: $xmm0 = VMOVSLDUPrm $rdi, 1, $noreg, 0, $noreg + $xmm0 = VMOVSLDUPZ128rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $xmm0 = VMOVSLDUPrr $xmm0 + $xmm0 = VMOVSLDUPZ128rr $xmm0 ; CHECK: $xmm0 = VPSHUFBrm $xmm0, $rdi, 1, $noreg, 0, $noreg $xmm0 = VPSHUFBZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSHUFBrr $xmm0, $xmm1 - $xmm0 = VPSHUFBZ128rr $xmm0, $xmm1 + $xmm0 = VPSHUFBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSHUFDmi $rdi, 1, $noreg, 0, $noreg, -24 $xmm0 = VPSHUFDZ128mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $xmm0 = VPSHUFDri $xmm0, -24 - $xmm0 = VPSHUFDZ128ri $xmm0, -24 + $xmm0 = VPSHUFDZ128ri $xmm0, -24 ; CHECK: $xmm0 = VPSHUFHWmi $rdi, 1, $noreg, 0, $noreg, -24 $xmm0 = VPSHUFHWZ128mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $xmm0 = VPSHUFHWri $xmm0, -24 - $xmm0 = VPSHUFHWZ128ri $xmm0, -24 + $xmm0 = VPSHUFHWZ128ri $xmm0, -24 ; CHECK: $xmm0 = VPSHUFLWmi $rdi, 1, $noreg, 0, $noreg, -24 $xmm0 = VPSHUFLWZ128mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $xmm0 = VPSHUFLWri $xmm0, -24 - $xmm0 = VPSHUFLWZ128ri $xmm0, -24 + $xmm0 = VPSHUFLWZ128ri $xmm0, -24 ; CHECK: $xmm0 = VPSLLDQri $xmm0, 7 $xmm0 = VPSLLDQZ128ri $xmm0, 7 ; CHECK: $xmm0 = VSHUFPDrmi $xmm0, $rip, 1, $noreg, 0, $noreg, -24 @@ -1697,28 +1697,28 @@ body: | $xmm0 = VSHUFPSZ128rri $xmm0, $xmm1, -24 ; CHECK: $xmm0 = VPSADBWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSADBWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPSADBWrr $xmm0, $xmm1 - $xmm0 = VPSADBWZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPSADBWrr $xmm0, $xmm1 + $xmm0 = VPSADBWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VBROADCASTSSrm $rip, 1, $noreg, 0, $noreg $xmm0 = VBROADCASTSSZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VBROADCASTSSrr $xmm0 - $xmm0 = VBROADCASTSSZ128rr $xmm0 + $xmm0 = VBROADCASTSSZ128rr $xmm0 ; CHECK: $xmm0 = VPBROADCASTBrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPBROADCASTBZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPBROADCASTBrr $xmm0 - $xmm0 = VPBROADCASTBZ128rr $xmm0 + $xmm0 = VPBROADCASTBZ128rr $xmm0 ; CHECK: $xmm0 = VPBROADCASTDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPBROADCASTDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPBROADCASTDrr $xmm0 - $xmm0 = VPBROADCASTDZ128rr $xmm0 + $xmm0 = VPBROADCASTDZ128rr $xmm0 ; CHECK: $xmm0 = VPBROADCASTQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPBROADCASTQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPBROADCASTQrr $xmm0 - $xmm0 = VPBROADCASTQZ128rr $xmm0 + $xmm0 = VPBROADCASTQZ128rr $xmm0 ; CHECK: $xmm0 = VPBROADCASTWrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPBROADCASTWZ128rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPBROADCASTWrr $xmm0 - $xmm0 = VPBROADCASTWZ128rr $xmm0 + ; CHECK: $xmm0 = VPBROADCASTWrr $xmm0 + $xmm0 = VPBROADCASTWZ128rr $xmm0 ; CHECK: $xmm0 = VPBROADCASTQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VBROADCASTI32X2Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPBROADCASTQrr $xmm0 @@ -1730,15 +1730,15 @@ body: | ; CHECK: $xmm0 = VPABSBrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPABSBZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPABSBrr $xmm0 - $xmm0 = VPABSBZ128rr $xmm0 + $xmm0 = VPABSBZ128rr $xmm0 ; CHECK: $xmm0 = VPABSDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPABSDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPABSDrr $xmm0 - $xmm0 = VPABSDZ128rr $xmm0 + $xmm0 = VPABSDZ128rr $xmm0 ; CHECK: $xmm0 = VPABSWrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPABSWZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPABSWrr $xmm0 - $xmm0 = VPABSWZ128rr $xmm0 + $xmm0 = VPABSWZ128rr $xmm0 ; CHECK: $xmm0 = VPALIGNRrmi $xmm0, $rip, 1, $noreg, 0, $noreg, 15 $xmm0 = VPALIGNRZ128rmi $xmm0, $rip, 1, $noreg, 0, $noreg, 15 ; CHECK: $xmm0 = VPALIGNRrri $xmm0, $xmm1, 15 @@ -2074,38 +2074,38 @@ body: | $xmm0 = VFNMSUB231SSZr $xmm0, $xmm1, $xmm2, implicit $mxcsr ; CHECK: $xmm0 = VFNMSUB231SSr_Int $xmm0, $xmm1, $xmm2, implicit $mxcsr $xmm0 = VFNMSUB231SSZr_Int $xmm0, $xmm1, $xmm2, implicit $mxcsr - ; CHECK: VPEXTRBmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - VPEXTRBZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - ; CHECK: $eax = VPEXTRBrr $xmm0, 1 - $eax = VPEXTRBZrr $xmm0, 1 - ; CHECK: VPEXTRDmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - VPEXTRDZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - ; CHECK: $eax = VPEXTRDrr $xmm0, 1 - $eax = VPEXTRDZrr $xmm0, 1 - ; CHECK: VPEXTRQmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - VPEXTRQZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - ; CHECK: $rax = VPEXTRQrr $xmm0, 1 - $rax = VPEXTRQZrr $xmm0, 1 - ; CHECK: VPEXTRWmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - VPEXTRWZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - ; CHECK: $eax = VPEXTRWrr $xmm0, 1 - $eax = VPEXTRWZrr $xmm0, 1 - ; CHECK: $xmm0 = VPINSRBrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm0 = VPINSRBZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - ; CHECK: $xmm0 = VPINSRBrr $xmm0, $edi, 5 - $xmm0 = VPINSRBZrr $xmm0, $edi, 5 - ; CHECK: $xmm0 = VPINSRDrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm0 = VPINSRDZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - ; CHECK: $xmm0 = VPINSRDrr $xmm0, $edi, 5 - $xmm0 = VPINSRDZrr $xmm0, $edi, 5 - ; CHECK: $xmm0 = VPINSRQrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm0 = VPINSRQZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - ; CHECK: $xmm0 = VPINSRQrr $xmm0, $rdi, 5 - $xmm0 = VPINSRQZrr $xmm0, $rdi, 5 - ; CHECK: $xmm0 = VPINSRWrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm0 = VPINSRWZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - ; CHECK: $xmm0 = VPINSRWrr $xmm0, $edi, 5 - $xmm0 = VPINSRWZrr $xmm0, $edi, 5 + ; CHECK: VPEXTRBmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + VPEXTRBZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + ; CHECK: $eax = VPEXTRBrr $xmm0, 1 + $eax = VPEXTRBZrr $xmm0, 1 + ; CHECK: VPEXTRDmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + VPEXTRDZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + ; CHECK: $eax = VPEXTRDrr $xmm0, 1 + $eax = VPEXTRDZrr $xmm0, 1 + ; CHECK: VPEXTRQmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + VPEXTRQZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + ; CHECK: $rax = VPEXTRQrr $xmm0, 1 + $rax = VPEXTRQZrr $xmm0, 1 + ; CHECK: VPEXTRWmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + VPEXTRWZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + ; CHECK: $eax = VPEXTRWrr $xmm0, 1 + $eax = VPEXTRWZrr $xmm0, 1 + ; CHECK: $xmm0 = VPINSRBrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm0 = VPINSRBZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: $xmm0 = VPINSRBrr $xmm0, $edi, 5 + $xmm0 = VPINSRBZrr $xmm0, $edi, 5 + ; CHECK: $xmm0 = VPINSRDrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm0 = VPINSRDZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: $xmm0 = VPINSRDrr $xmm0, $edi, 5 + $xmm0 = VPINSRDZrr $xmm0, $edi, 5 + ; CHECK: $xmm0 = VPINSRQrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm0 = VPINSRQZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: $xmm0 = VPINSRQrr $xmm0, $rdi, 5 + $xmm0 = VPINSRQZrr $xmm0, $rdi, 5 + ; CHECK: $xmm0 = VPINSRWrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm0 = VPINSRWZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: $xmm0 = VPINSRWrr $xmm0, $edi, 5 + $xmm0 = VPINSRWZrr $xmm0, $edi, 5 ; CHECK: $xmm0 = VSQRTSDm $xmm0, $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VSQRTSDZm $xmm0, $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VSQRTSDm_Int $xmm0, $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr @@ -2135,9 +2135,9 @@ body: | ; CHECK: $xmm0 = VCVTSD2SSrr_Int $xmm0, $xmm1, implicit $mxcsr $xmm0 = VCVTSD2SSZrr_Int $xmm0, $xmm1, implicit $mxcsr ; CHECK: $xmm0 = VCVTSI2SDrm $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VCVTSI2SDZrm $xmm0, $rdi, 1, $noreg, 0, $noreg + $xmm0 = VCVTSI2SDZrm $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VCVTSI2SDrm_Int $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VCVTSI2SDZrm_Int $xmm0, $rdi, 1, $noreg, 0, $noreg + $xmm0 = VCVTSI2SDZrm_Int $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VCVTSI2SDrr $xmm0, $edi $xmm0 = VCVTSI2SDZrr $xmm0, $edi ; CHECK: $xmm0 = VCVTSI2SDrr_Int $xmm0, $edi @@ -2214,10 +2214,10 @@ body: | $edi = VCVTTSS2SIZrr $xmm0, implicit $mxcsr ; CHECK: $edi = VCVTTSS2SIrr_Int $xmm0, implicit $mxcsr $edi = VCVTTSS2SIZrr_Int $xmm0, implicit $mxcsr - ; CHECK: $xmm0 = VMOV64toSDrr $rdi - $xmm0 = VMOV64toSDZrr $rdi + ; CHECK: $xmm0 = VMOV64toSDrr $rdi + $xmm0 = VMOV64toSDZrr $rdi ; CHECK: $xmm0 = VMOVDI2SSrr $eax - $xmm0 = VMOVDI2SSZrr $eax + $xmm0 = VMOVDI2SSZrr $eax ; CHECK: VMOVSDmr $rdi, 1, $noreg, 0, $noreg, $xmm0 VMOVSDZmr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVSDrm $rip, 1, $noreg, 0, $noreg @@ -2239,15 +2239,15 @@ body: | ; CHECK: $eax = VMOVSS2DIrr $xmm0 $eax = VMOVSS2DIZrr $xmm0 ; CHECK: $xmm0 = VMOV64toPQIrr $rdi - $xmm0 = VMOV64toPQIZrr $rdi + $xmm0 = VMOV64toPQIZrr $rdi ; CHECK: $xmm0 = VMOV64toPQIrm $rdi, 1, $noreg, 0, $noreg $xmm0 = VMOV64toPQIZrm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VMOV64toSDrr $rdi - $xmm0 = VMOV64toSDZrr $rdi + ; CHECK: $xmm0 = VMOV64toSDrr $rdi + $xmm0 = VMOV64toSDZrr $rdi ; CHECK: $xmm0 = VMOVDI2PDIrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVDI2PDIZrm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVDI2PDIrr $edi - $xmm0 = VMOVDI2PDIZrr $edi + $xmm0 = VMOVDI2PDIZrr $edi ; CHECK: $xmm0 = VMOVLHPSrr $xmm0, $xmm1 $xmm0 = VMOVLHPSZrr $xmm0, $xmm1 ; CHECK: $xmm0 = VMOVHLPSrr $xmm0, $xmm1 @@ -2261,13 +2261,13 @@ body: | ; CHECK: VMOVPQI2QImr $rdi, 1, $noreg, 0, $noreg, $xmm0 VMOVPQI2QIZmr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $rdi = VMOVPQIto64rr $xmm0 - $rdi = VMOVPQIto64Zrr $xmm0 + $rdi = VMOVPQIto64Zrr $xmm0 ; CHECK: VMOVPQIto64mr $rdi, 1, $noreg, 0, $noreg, $xmm0 VMOVPQIto64Zmr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVQI2PQIrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVQI2PQIZrm $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VMOVZPQILo2PQIrr $xmm0 - $xmm0 = VMOVZPQILo2PQIZrr $xmm0 + ; CHECK: $xmm0 = VMOVZPQILo2PQIrr $xmm0 + $xmm0 = VMOVZPQILo2PQIZrr $xmm0 ; CHECK: VCOMISDrm_Int $xmm0, $rdi, 1, $noreg, 0, $noreg, implicit-def $eflags, implicit $mxcsr VCOMISDZrm_Int $xmm0, $rdi, 1, $noreg, 0, $noreg, implicit-def $eflags, implicit $mxcsr ; CHECK: VCOMISDrr_Int $xmm0, $xmm1, implicit-def $eflags, implicit $mxcsr @@ -2335,137 +2335,137 @@ name: evex_z256_to_evex_test body: | bb.0: ; CHECK: VMOVAPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVAPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVAPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVAPDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVAPDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVAPDZ256rr $ymm16 - $ymm16 = VMOVAPDZ256rr $ymm16 + $ymm16 = VMOVAPDZ256rr $ymm16 ; CHECK: VMOVAPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVAPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVAPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVAPSZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVAPSZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVAPSZ256rr $ymm16 - $ymm16 = VMOVAPSZ256rr $ymm16 + $ymm16 = VMOVAPSZ256rr $ymm16 ; CHECK: $ymm16 = VMOVDDUPZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVDDUPZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVDDUPZ256rr $ymm16 - $ymm16 = VMOVDDUPZ256rr $ymm16 + $ymm16 = VMOVDDUPZ256rr $ymm16 ; CHECK: VMOVDQA32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVDQA32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVDQA32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVDQA32Z256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVDQA32Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVDQA32Z256rr $ymm16 - $ymm16 = VMOVDQA32Z256rr $ymm16 + $ymm16 = VMOVDQA32Z256rr $ymm16 ; CHECK: VMOVDQA64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVDQA64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVDQA64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVDQA64Z256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVDQA64Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVDQA64Z256rr $ymm16 - $ymm16 = VMOVDQA64Z256rr $ymm16 + $ymm16 = VMOVDQA64Z256rr $ymm16 ; CHECK: VMOVDQU16Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVDQU16Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVDQU16Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVDQU16Z256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVDQU16Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVDQU16Z256rr $ymm16 - $ymm16 = VMOVDQU16Z256rr $ymm16 + $ymm16 = VMOVDQU16Z256rr $ymm16 ; CHECK: VMOVDQU32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVDQU32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVDQU32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVDQU32Z256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVDQU32Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVDQU32Z256rr $ymm16 - $ymm16 = VMOVDQU32Z256rr $ymm16 + $ymm16 = VMOVDQU32Z256rr $ymm16 ; CHECK: VMOVDQU64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVDQU64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVDQU64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVDQU64Z256rm $rip, 1, $noreg, 0, $noreg - $ymm16 = VMOVDQU64Z256rm $rip, 1, $noreg, 0, $noreg + $ymm16 = VMOVDQU64Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVDQU64Z256rr $ymm16 - $ymm16 = VMOVDQU64Z256rr $ymm16 + $ymm16 = VMOVDQU64Z256rr $ymm16 ; CHECK: VMOVDQU8Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVDQU8Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVDQU8Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVDQU8Z256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVDQU8Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVDQU8Z256rr $ymm16 - $ymm16 = VMOVDQU8Z256rr $ymm16 + $ymm16 = VMOVDQU8Z256rr $ymm16 ; CHECK: $ymm16 = VMOVNTDQAZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVNTDQAZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: VMOVNTDQZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVNTDQZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVNTDQZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: VMOVNTPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVNTPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVNTPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: VMOVNTPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVNTPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVNTPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVSHDUPZ256rm $rip, 1, $noreg, 0, $noreg - $ymm16 = VMOVSHDUPZ256rm $rip, 1, $noreg, 0, $noreg + $ymm16 = VMOVSHDUPZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVSHDUPZ256rr $ymm16 - $ymm16 = VMOVSHDUPZ256rr $ymm16 + $ymm16 = VMOVSHDUPZ256rr $ymm16 ; CHECK: $ymm16 = VMOVSLDUPZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVSLDUPZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVSLDUPZ256rr $ymm16 - $ymm16 = VMOVSLDUPZ256rr $ymm16 + $ymm16 = VMOVSLDUPZ256rr $ymm16 ; CHECK: VMOVUPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVUPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVUPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVUPDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVUPDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVUPDZ256rr $ymm16 - $ymm16 = VMOVUPDZ256rr $ymm16 - ; CHECK: VMOVUPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVUPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + $ymm16 = VMOVUPDZ256rr $ymm16 + ; CHECK: VMOVUPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVUPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VPANDDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPANDDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPANDDZ256rr $ymm16, $ymm1 - $ymm16 = VPANDDZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPANDDZ256rr $ymm16, $ymm1 + $ymm16 = VPANDDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPANDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPANDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPANDQZ256rr $ymm16, $ymm1 - $ymm16 = VPANDQZ256rr $ymm16, $ymm1 + $ymm16 = VPANDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPANDNDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPANDNDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPANDNDZ256rr $ymm16, $ymm1 - $ymm16 = VPANDNDZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPANDNDZ256rr $ymm16, $ymm1 + $ymm16 = VPANDNDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPANDNQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPANDNQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPANDNQZ256rr $ymm16, $ymm1 - $ymm16 = VPANDNQZ256rr $ymm16, $ymm1 + $ymm16 = VPANDNQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPAVGBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPAVGBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPAVGBZ256rr $ymm16, $ymm1 - $ymm16 = VPAVGBZ256rr $ymm16, $ymm1 + $ymm16 = VPAVGBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPAVGWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPAVGWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPAVGWZ256rr $ymm16, $ymm1 - $ymm16 = VPAVGWZ256rr $ymm16, $ymm1 + $ymm16 = VPAVGWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPADDBZ256rr $ymm16, $ymm1 - $ymm16 = VPADDBZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPADDBZ256rr $ymm16, $ymm1 + $ymm16 = VPADDBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPADDDZ256rr $ymm16, $ymm1 - $ymm16 = VPADDDZ256rr $ymm16, $ymm1 + $ymm16 = VPADDDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPADDQZ256rr $ymm16, $ymm1 - $ymm16 = VPADDQZ256rr $ymm16, $ymm1 + $ymm16 = VPADDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPADDSBZ256rr $ymm16, $ymm1 - $ymm16 = VPADDSBZ256rr $ymm16, $ymm1 + $ymm16 = VPADDSBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPADDSWZ256rr $ymm16, $ymm1 - $ymm16 = VPADDSWZ256rr $ymm16, $ymm1 + $ymm16 = VPADDSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDUSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDUSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPADDUSBZ256rr $ymm16, $ymm1 - $ymm16 = VPADDUSBZ256rr $ymm16, $ymm1 + $ymm16 = VPADDUSBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDUSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDUSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPADDUSWZ256rr $ymm16, $ymm1 - $ymm16 = VPADDUSWZ256rr $ymm16, $ymm1 + $ymm16 = VPADDUSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPADDWZ256rr $ymm16, $ymm1 - $ymm16 = VPADDWZ256rr $ymm16, $ymm1 + $ymm16 = VPADDWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VMULPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm16 = VMULPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm16 = VMULPDZ256rr $ymm16, $ymm1, implicit $mxcsr @@ -2477,143 +2477,143 @@ body: | ; CHECK: $ymm16 = VORPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VORPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VORPDZ256rr $ymm16, $ymm1 - $ymm16 = VORPDZ256rr $ymm16, $ymm1 + $ymm16 = VORPDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VORPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VORPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VORPSZ256rr $ymm16, $ymm1 - $ymm16 = VORPSZ256rr $ymm16, $ymm1 + $ymm16 = VORPSZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMADDUBSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMADDUBSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMADDUBSWZ256rr $ymm16, $ymm1 - $ymm16 = VPMADDUBSWZ256rr $ymm16, $ymm1 + $ymm16 = VPMADDUBSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMADDWDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMADDWDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMADDWDZ256rr $ymm16, $ymm1 - $ymm16 = VPMADDWDZ256rr $ymm16, $ymm1 + $ymm16 = VPMADDWDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMAXSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMAXSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMAXSBZ256rr $ymm16, $ymm1 - $ymm16 = VPMAXSBZ256rr $ymm16, $ymm1 + $ymm16 = VPMAXSBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMAXSDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMAXSDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMAXSDZ256rr $ymm16, $ymm1 - $ymm16 = VPMAXSDZ256rr $ymm16, $ymm1 + $ymm16 = VPMAXSDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMAXSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMAXSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMAXSWZ256rr $ymm16, $ymm1 - $ymm16 = VPMAXSWZ256rr $ymm16, $ymm1 + $ymm16 = VPMAXSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMAXUBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMAXUBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMAXUBZ256rr $ymm16, $ymm1 - $ymm16 = VPMAXUBZ256rr $ymm16, $ymm1 + $ymm16 = VPMAXUBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMAXUDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMAXUDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMAXUDZ256rr $ymm16, $ymm1 - $ymm16 = VPMAXUDZ256rr $ymm16, $ymm1 + $ymm16 = VPMAXUDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMAXUWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMAXUWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMAXUWZ256rr $ymm16, $ymm1 - $ymm16 = VPMAXUWZ256rr $ymm16, $ymm1 + $ymm16 = VPMAXUWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMINSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMINSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMINSBZ256rr $ymm16, $ymm1 - $ymm16 = VPMINSBZ256rr $ymm16, $ymm1 + $ymm16 = VPMINSBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMINSDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMINSDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMINSDZ256rr $ymm16, $ymm1 - $ymm16 = VPMINSDZ256rr $ymm16, $ymm1 + $ymm16 = VPMINSDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMINSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMINSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMINSWZ256rr $ymm16, $ymm1 - $ymm16 = VPMINSWZ256rr $ymm16, $ymm1 + $ymm16 = VPMINSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMINUBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMINUBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMINUBZ256rr $ymm16, $ymm1 - $ymm16 = VPMINUBZ256rr $ymm16, $ymm1 + $ymm16 = VPMINUBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMINUDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMINUDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMINUDZ256rr $ymm16, $ymm1 - $ymm16 = VPMINUDZ256rr $ymm16, $ymm1 + $ymm16 = VPMINUDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMINUWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMINUWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMINUWZ256rr $ymm16, $ymm1 - $ymm16 = VPMINUWZ256rr $ymm16, $ymm1 + $ymm16 = VPMINUWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMULDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMULDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPMULDQZ256rr $ymm16, $ymm1 - $ymm16 = VPMULDQZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPMULDQZ256rr $ymm16, $ymm1 + $ymm16 = VPMULDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMULHRSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMULHRSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMULHRSWZ256rr $ymm16, $ymm1 - $ymm16 = VPMULHRSWZ256rr $ymm16, $ymm1 + $ymm16 = VPMULHRSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMULHUWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMULHUWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMULHUWZ256rr $ymm16, $ymm1 - $ymm16 = VPMULHUWZ256rr $ymm16, $ymm1 + $ymm16 = VPMULHUWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMULHWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMULHWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMULHWZ256rr $ymm16, $ymm1 - $ymm16 = VPMULHWZ256rr $ymm16, $ymm1 + $ymm16 = VPMULHWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMULLDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMULLDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMULLDZ256rr $ymm16, $ymm1 - $ymm16 = VPMULLDZ256rr $ymm16, $ymm1 + $ymm16 = VPMULLDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMULLWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMULLWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPMULLWZ256rr $ymm16, $ymm1 - $ymm16 = VPMULLWZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPMULLWZ256rr $ymm16, $ymm1 + $ymm16 = VPMULLWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMULUDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMULUDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMULUDQZ256rr $ymm16, $ymm1 - $ymm16 = VPMULUDQZ256rr $ymm16, $ymm1 + $ymm16 = VPMULUDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPORDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPORDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPORDZ256rr $ymm16, $ymm1 - $ymm16 = VPORDZ256rr $ymm16, $ymm1 + $ymm16 = VPORDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPORQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPORQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPORQZ256rr $ymm16, $ymm1 - $ymm16 = VPORQZ256rr $ymm16, $ymm1 + $ymm16 = VPORQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBBZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBBZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBDZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBDZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBQZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBQZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBSBZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBSBZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBSBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBSWZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBSWZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBUSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBUSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBUSBZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBUSBZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBUSBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBUSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBUSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBUSWZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBUSWZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBUSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBWZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBWZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPXORDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPXORDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPXORDZ256rr $ymm16, $ymm1 - $ymm16 = VPXORDZ256rr $ymm16, $ymm1 + $ymm16 = VPXORDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPXORQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPXORQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPXORQZ256rr $ymm16, $ymm1 - $ymm16 = VPXORQZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPXORQZ256rr $ymm16, $ymm1 + $ymm16 = VPXORQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VADDPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm16 = VADDPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm16 = VADDPDZ256rr $ymm16, $ymm1, implicit $mxcsr @@ -2625,19 +2625,19 @@ body: | ; CHECK: $ymm16 = VANDNPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VANDNPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VANDNPDZ256rr $ymm16, $ymm1 - $ymm16 = VANDNPDZ256rr $ymm16, $ymm1 + $ymm16 = VANDNPDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VANDNPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VANDNPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VANDNPSZ256rr $ymm16, $ymm1 - $ymm16 = VANDNPSZ256rr $ymm16, $ymm1 + $ymm16 = VANDNPSZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VANDPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VANDPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VANDPDZ256rr $ymm16, $ymm1 - $ymm16 = VANDPDZ256rr $ymm16, $ymm1 + $ymm16 = VANDPDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VANDPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VANDPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VANDPSZ256rr $ymm16, $ymm1 - $ymm16 = VANDPSZ256rr $ymm16, $ymm1 + $ymm16 = VANDPSZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VDIVPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm16 = VDIVPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm16 = VDIVPDZ256rr $ymm16, $ymm1, implicit $mxcsr @@ -2681,43 +2681,43 @@ body: | ; CHECK: $ymm16 = VXORPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VXORPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VXORPDZ256rr $ymm16, $ymm1 - $ymm16 = VXORPDZ256rr $ymm16, $ymm1 + $ymm16 = VXORPDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VXORPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VXORPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VXORPSZ256rr $ymm16, $ymm1 - $ymm16 = VXORPSZ256rr $ymm16, $ymm1 + $ymm16 = VXORPSZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPACKSSDWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPACKSSDWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPACKSSDWZ256rr $ymm16, $ymm1 - $ymm16 = VPACKSSDWZ256rr $ymm16, $ymm1 + $ymm16 = VPACKSSDWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPACKSSWBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPACKSSWBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPACKSSWBZ256rr $ymm16, $ymm1 - $ymm16 = VPACKSSWBZ256rr $ymm16, $ymm1 + $ymm16 = VPACKSSWBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPACKUSDWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPACKUSDWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPACKUSDWZ256rr $ymm16, $ymm1 - $ymm16 = VPACKUSDWZ256rr $ymm16, $ymm1 + $ymm16 = VPACKUSDWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPACKUSWBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPACKUSWBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPACKUSWBZ256rr $ymm16, $ymm1 - $ymm16 = VPACKUSWBZ256rr $ymm16, $ymm1 + $ymm16 = VPACKUSWBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VUNPCKHPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VUNPCKHPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VUNPCKHPDZ256rr $ymm16, $ymm1 - $ymm16 = VUNPCKHPDZ256rr $ymm16, $ymm1 + $ymm16 = VUNPCKHPDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VUNPCKHPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VUNPCKHPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VUNPCKHPSZ256rr $ymm16, $ymm1 - $ymm16 = VUNPCKHPSZ256rr $ymm16, $ymm1 + $ymm16 = VUNPCKHPSZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VUNPCKLPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VUNPCKLPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VUNPCKLPDZ256rr $ymm16, $ymm1 - $ymm16 = VUNPCKLPDZ256rr $ymm16, $ymm1 + $ymm16 = VUNPCKLPDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VUNPCKLPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VUNPCKLPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VUNPCKLPSZ256rr $ymm16, $ymm1 - $ymm16 = VUNPCKLPSZ256rr $ymm16, $ymm1 + $ymm16 = VUNPCKLPSZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VSUBPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm16 = VSUBPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm16 = VSUBPDZ256rr $ymm16, $ymm1, implicit $mxcsr @@ -2729,35 +2729,35 @@ body: | ; CHECK: $ymm16 = VPUNPCKHBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKHBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPUNPCKHBWZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKHBWZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKHBWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPUNPCKHDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKHDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPUNPCKHDQZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKHDQZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKHDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPUNPCKHQDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKHQDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPUNPCKHQDQZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKHQDQZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKHQDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPUNPCKHWDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKHWDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPUNPCKHWDZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKHWDZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKHWDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPUNPCKLBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKLBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPUNPCKLBWZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKLBWZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKLBWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPUNPCKLDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKLDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPUNPCKLDQZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKLDQZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPUNPCKLDQZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKLDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPUNPCKLQDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKLQDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPUNPCKLQDQZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKLQDQZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPUNPCKLQDQZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKLQDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPUNPCKLWDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKLWDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPUNPCKLWDZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKLWDZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPUNPCKLWDZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKLWDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VFMADD132PDZ256m $ymm16, $ymm16, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm16 = VFMADD132PDZ256m $ymm16, $ymm16, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm16 = VFMADD132PDZ256r $ymm16, $ymm1, $ymm2, implicit $mxcsr @@ -2903,97 +2903,97 @@ body: | ; CHECK: $ymm16 = VFNMSUB231PSZ256r $ymm16, $ymm1, $ymm2, implicit $mxcsr $ymm16 = VFNMSUB231PSZ256r $ymm16, $ymm1, $ymm2, implicit $mxcsr ; CHECK: $ymm16 = VPSRADZ256ri $ymm16, 7 - $ymm16 = VPSRADZ256ri $ymm16, 7 + $ymm16 = VPSRADZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSRADZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRADZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSRADZ256rr $ymm16, $xmm1 - $ymm16 = VPSRADZ256rr $ymm16, $xmm1 + $ymm16 = VPSRADZ256rr $ymm16, $xmm1 ; CHECK: $ymm16 = VPSRAVDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRAVDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSRAVDZ256rr $ymm16, $ymm1 - $ymm16 = VPSRAVDZ256rr $ymm16, $ymm1 + $ymm16 = VPSRAVDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSRAWZ256ri $ymm16, 7 - $ymm16 = VPSRAWZ256ri $ymm16, 7 + $ymm16 = VPSRAWZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSRAWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRAWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSRAWZ256rr $ymm16, $xmm1 - $ymm16 = VPSRAWZ256rr $ymm16, $xmm1 + $ymm16 = VPSRAWZ256rr $ymm16, $xmm1 ; CHECK: $ymm16 = VPSRLDQZ256ri $ymm16, 7 $ymm16 = VPSRLDQZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSRLDZ256ri $ymm16, 7 - $ymm16 = VPSRLDZ256ri $ymm16, 7 + $ymm16 = VPSRLDZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSRLDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRLDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSRLDZ256rr $ymm16, $xmm1 - $ymm16 = VPSRLDZ256rr $ymm16, $xmm1 + $ymm16 = VPSRLDZ256rr $ymm16, $xmm1 ; CHECK: $ymm16 = VPSRLQZ256ri $ymm16, 7 - $ymm16 = VPSRLQZ256ri $ymm16, 7 + $ymm16 = VPSRLQZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSRLQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRLQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSRLQZ256rr $ymm16, $xmm1 - $ymm16 = VPSRLQZ256rr $ymm16, $xmm1 + $ymm16 = VPSRLQZ256rr $ymm16, $xmm1 ; CHECK: $ymm16 = VPSRLVDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRLVDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSRLVDZ256rr $ymm16, $ymm1 - $ymm16 = VPSRLVDZ256rr $ymm16, $ymm1 + $ymm16 = VPSRLVDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSRLVQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRLVQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSRLVQZ256rr $ymm16, $ymm1 - $ymm16 = VPSRLVQZ256rr $ymm16, $ymm1 + $ymm16 = VPSRLVQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSRLWZ256ri $ymm16, 7 - $ymm16 = VPSRLWZ256ri $ymm16, 7 + $ymm16 = VPSRLWZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSRLWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRLWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPSRLWZ256rr $ymm16, $xmm1 - $ymm16 = VPSRLWZ256rr $ymm16, $xmm1 + ; CHECK: $ymm16 = VPSRLWZ256rr $ymm16, $xmm1 + $ymm16 = VPSRLWZ256rr $ymm16, $xmm1 ; CHECK: $ymm16 = VPMOVSXBDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVSXBDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVSXBDZ256rr $xmm0 - $ymm16 = VPMOVSXBDZ256rr $xmm0 + $ymm16 = VPMOVSXBDZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVSXBQZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVSXBQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVSXBQZ256rr $xmm0 - $ymm16 = VPMOVSXBQZ256rr $xmm0 + $ymm16 = VPMOVSXBQZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVSXBWZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVSXBWZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVSXBWZ256rr $xmm0 - $ymm16 = VPMOVSXBWZ256rr $xmm0 + $ymm16 = VPMOVSXBWZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVSXDQZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVSXDQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVSXDQZ256rr $xmm0 - $ymm16 = VPMOVSXDQZ256rr $xmm0 + $ymm16 = VPMOVSXDQZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVSXWDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVSXWDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVSXWDZ256rr $xmm0 - $ymm16 = VPMOVSXWDZ256rr $xmm0 + $ymm16 = VPMOVSXWDZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVSXWQZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVSXWQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVSXWQZ256rr $xmm0 - $ymm16 = VPMOVSXWQZ256rr $xmm0 + $ymm16 = VPMOVSXWQZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVZXBDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVZXBDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVZXBDZ256rr $xmm0 - $ymm16 = VPMOVZXBDZ256rr $xmm0 + $ymm16 = VPMOVZXBDZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVZXBQZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVZXBQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVZXBQZ256rr $xmm0 - $ymm16 = VPMOVZXBQZ256rr $xmm0 + $ymm16 = VPMOVZXBQZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVZXBWZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVZXBWZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVZXBWZ256rr $xmm0 - $ymm16 = VPMOVZXBWZ256rr $xmm0 + $ymm16 = VPMOVZXBWZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVZXDQZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVZXDQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVZXDQZ256rr $xmm0 - $ymm16 = VPMOVZXDQZ256rr $xmm0 + $ymm16 = VPMOVZXDQZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVZXWDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVZXWDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVZXWDZ256rr $xmm0 - $ymm16 = VPMOVZXWDZ256rr $xmm0 + $ymm16 = VPMOVZXWDZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVZXWQZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVZXWQZ256rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPMOVZXWQZ256rr $xmm0 - $ymm16 = VPMOVZXWQZ256rr $xmm0 + ; CHECK: $ymm16 = VPMOVZXWQZ256rr $xmm0 + $ymm16 = VPMOVZXWQZ256rr $xmm0 ; CHECK: $ymm16 = VBROADCASTF32X2Z256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VBROADCASTF32X2Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VBROADCASTF32X2Z256rr $xmm16 @@ -3003,23 +3003,23 @@ body: | ; CHECK: $ymm16 = VBROADCASTSDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VBROADCASTSDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VBROADCASTSDZ256rr $xmm0 - $ymm16 = VBROADCASTSDZ256rr $xmm0 + $ymm16 = VBROADCASTSDZ256rr $xmm0 ; CHECK: $ymm16 = VBROADCASTSSZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VBROADCASTSSZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VBROADCASTSSZ256rr $xmm0 - $ymm16 = VBROADCASTSSZ256rr $xmm0 + $ymm16 = VBROADCASTSSZ256rr $xmm0 ; CHECK: $ymm16 = VPBROADCASTBZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPBROADCASTBZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPBROADCASTBZ256rr $xmm0 - $ymm16 = VPBROADCASTBZ256rr $xmm0 + $ymm16 = VPBROADCASTBZ256rr $xmm0 ; CHECK: $ymm16 = VPBROADCASTDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPBROADCASTDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPBROADCASTDZ256rr $xmm0 - $ymm16 = VPBROADCASTDZ256rr $xmm0 + $ymm16 = VPBROADCASTDZ256rr $xmm0 ; CHECK: $ymm16 = VPBROADCASTWZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPBROADCASTWZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPBROADCASTWZ256rr $xmm0 - $ymm16 = VPBROADCASTWZ256rr $xmm0 + $ymm16 = VPBROADCASTWZ256rr $xmm0 ; CHECK: $ymm16 = VBROADCASTI32X4Z256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VBROADCASTI32X4Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VBROADCASTI32X2Z256rm $rip, 1, $noreg, 0, $noreg @@ -3028,66 +3028,66 @@ body: | $ymm16 = VBROADCASTI32X2Z256rr $xmm16 ; CHECK: $ymm16 = VPBROADCASTQZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPBROADCASTQZ256rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPBROADCASTQZ256rr $xmm0 - $ymm16 = VPBROADCASTQZ256rr $xmm0 + ; CHECK: $ymm16 = VPBROADCASTQZ256rr $xmm0 + $ymm16 = VPBROADCASTQZ256rr $xmm0 ; CHECK: $ymm16 = VPABSBZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPABSBZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPABSBZ256rr $ymm16 - $ymm16 = VPABSBZ256rr $ymm16 + $ymm16 = VPABSBZ256rr $ymm16 ; CHECK: $ymm16 = VPABSDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPABSDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPABSDZ256rr $ymm16 - $ymm16 = VPABSDZ256rr $ymm16 + $ymm16 = VPABSDZ256rr $ymm16 ; CHECK: $ymm16 = VPABSWZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPABSWZ256rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPABSWZ256rr $ymm16 - $ymm16 = VPABSWZ256rr $ymm16 + ; CHECK: $ymm16 = VPABSWZ256rr $ymm16 + $ymm16 = VPABSWZ256rr $ymm16 ; CHECK: $ymm16 = VPSADBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSADBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPSADBWZ256rr $ymm16, $ymm1 - $ymm16 = VPSADBWZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPSADBWZ256rr $ymm16, $ymm1 + $ymm16 = VPSADBWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPERMDZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg - $ymm16 = VPERMDZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg + $ymm16 = VPERMDZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPERMDZ256rr $ymm1, $ymm16 - $ymm16 = VPERMDZ256rr $ymm1, $ymm16 + $ymm16 = VPERMDZ256rr $ymm1, $ymm16 ; CHECK: $ymm16 = VPERMILPDZ256mi $rdi, 1, $noreg, 0, $noreg, 7 $ymm16 = VPERMILPDZ256mi $rdi, 1, $noreg, 0, $noreg, 7 ; CHECK: $ymm16 = VPERMILPDZ256ri $ymm16, 7 - $ymm16 = VPERMILPDZ256ri $ymm16, 7 + $ymm16 = VPERMILPDZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPERMILPDZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg - $ymm16 = VPERMILPDZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg + $ymm16 = VPERMILPDZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPERMILPDZ256rr $ymm1, $ymm16 - $ymm16 = VPERMILPDZ256rr $ymm1, $ymm16 + $ymm16 = VPERMILPDZ256rr $ymm1, $ymm16 ; CHECK: $ymm16 = VPERMILPSZ256mi $rdi, 1, $noreg, 0, $noreg, 7 $ymm16 = VPERMILPSZ256mi $rdi, 1, $noreg, 0, $noreg, 7 ; CHECK: $ymm16 = VPERMILPSZ256ri $ymm16, 7 - $ymm16 = VPERMILPSZ256ri $ymm16, 7 + $ymm16 = VPERMILPSZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPERMILPSZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg - $ymm16 = VPERMILPSZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg + $ymm16 = VPERMILPSZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPERMILPSZ256rr $ymm1, $ymm16 - $ymm16 = VPERMILPSZ256rr $ymm1, $ymm16 + $ymm16 = VPERMILPSZ256rr $ymm1, $ymm16 ; CHECK: $ymm16 = VPERMPDZ256mi $rdi, 1, $noreg, 0, $noreg, 7 $ymm16 = VPERMPDZ256mi $rdi, 1, $noreg, 0, $noreg, 7 ; CHECK: $ymm16 = VPERMPDZ256ri $ymm16, 7 - $ymm16 = VPERMPDZ256ri $ymm16, 7 + $ymm16 = VPERMPDZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPERMPSZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg - $ymm16 = VPERMPSZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg + $ymm16 = VPERMPSZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPERMPSZ256rr $ymm1, $ymm16 - $ymm16 = VPERMPSZ256rr $ymm1, $ymm16 + $ymm16 = VPERMPSZ256rr $ymm1, $ymm16 ; CHECK: $ymm16 = VPERMQZ256mi $rdi, 1, $noreg, 0, $noreg, 7 $ymm16 = VPERMQZ256mi $rdi, 1, $noreg, 0, $noreg, 7 ; CHECK: $ymm16 = VPERMQZ256ri $ymm16, 7 - $ymm16 = VPERMQZ256ri $ymm16, 7 + $ymm16 = VPERMQZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSLLDQZ256ri $ymm16, 14 $ymm16 = VPSLLDQZ256ri $ymm16, 14 ; CHECK: $ymm16 = VPSLLDZ256ri $ymm16, 7 - $ymm16 = VPSLLDZ256ri $ymm16, 7 + $ymm16 = VPSLLDZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSLLDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSLLDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSLLDZ256rr $ymm16, $xmm16 $ymm16 = VPSLLDZ256rr $ymm16, $xmm16 ; CHECK: $ymm16 = VPSLLQZ256ri $ymm16, 7 - $ymm16 = VPSLLQZ256ri $ymm16, 7 + $ymm16 = VPSLLQZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSLLQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSLLQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSLLQZ256rr $ymm16, $xmm16 @@ -3101,7 +3101,7 @@ body: | ; CHECK: $ymm16 = VPSLLVQZ256rr $ymm16, $ymm16 $ymm16 = VPSLLVQZ256rr $ymm16, $ymm16 ; CHECK: $ymm16 = VPSLLWZ256ri $ymm16, 7 - $ymm16 = VPSLLWZ256ri $ymm16, 7 + $ymm16 = VPSLLWZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSLLWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSLLWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSLLWZ256rr $ymm16, $xmm16 @@ -3158,26 +3158,26 @@ body: | $ymm16 = VPALIGNRZ256rmi $ymm16, $rdi, 1, $noreg, 0, $noreg, 1 ; CHECK: $ymm16 = VPALIGNRZ256rri $ymm16, $ymm1, 1 $ymm16 = VPALIGNRZ256rri $ymm16, $ymm1, 1 - ; CHECK: $ymm16 = VMOVUPSZ256rm $rdi, 1, $noreg, 0, $noreg - $ymm16 = VMOVUPSZ256rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $ymm16 = VMOVUPSZ256rm $rdi, 1, $noreg, 0, $noreg + $ymm16 = VMOVUPSZ256rm $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVUPSZ256rr $ymm16 - $ymm16 = VMOVUPSZ256rr $ymm16 + $ymm16 = VMOVUPSZ256rr $ymm16 ; CHECK: $ymm16 = VPSHUFBZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg $ymm16 = VPSHUFBZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSHUFBZ256rr $ymm16, $ymm1 - $ymm16 = VPSHUFBZ256rr $ymm16, $ymm1 + $ymm16 = VPSHUFBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSHUFDZ256mi $rdi, 1, $noreg, 0, $noreg, -24 $ymm16 = VPSHUFDZ256mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm16 = VPSHUFDZ256ri $ymm16, -24 - $ymm16 = VPSHUFDZ256ri $ymm16, -24 + $ymm16 = VPSHUFDZ256ri $ymm16, -24 ; CHECK: $ymm16 = VPSHUFHWZ256mi $rdi, 1, $noreg, 0, $noreg, -24 $ymm16 = VPSHUFHWZ256mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm16 = VPSHUFHWZ256ri $ymm16, -24 - $ymm16 = VPSHUFHWZ256ri $ymm16, -24 + $ymm16 = VPSHUFHWZ256ri $ymm16, -24 ; CHECK: $ymm16 = VPSHUFLWZ256mi $rdi, 1, $noreg, 0, $noreg, -24 $ymm16 = VPSHUFLWZ256mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm16 = VPSHUFLWZ256ri $ymm16, -24 - $ymm16 = VPSHUFLWZ256ri $ymm16, -24 + $ymm16 = VPSHUFLWZ256ri $ymm16, -24 ; CHECK: $ymm16 = VSHUFPDZ256rmi $ymm16, $rip, 1, $noreg, 0, $noreg, -24 $ymm16 = VSHUFPDZ256rmi $ymm16, $rip, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm16 = VSHUFPDZ256rri $ymm16, $ymm1, -24 @@ -3220,8 +3220,8 @@ body: | $ymm16 = VSHUFI64X2Z256rri $ymm16, $ymm1, 228 RET64 -... ---- +... +--- # CHECK-LABEL: name: evex_z128_to_evex_test # CHECK: bb.0: @@ -3229,137 +3229,137 @@ name: evex_z128_to_evex_test body: | bb.0: ; CHECK: VMOVAPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVAPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVAPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVAPDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVAPDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVAPDZ128rr $xmm16 - $xmm16 = VMOVAPDZ128rr $xmm16 + $xmm16 = VMOVAPDZ128rr $xmm16 ; CHECK: VMOVAPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVAPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVAPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVAPSZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVAPSZ128rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VMOVAPSZ128rr $xmm16 - $xmm16 = VMOVAPSZ128rr $xmm16 + ; CHECK: $xmm16 = VMOVAPSZ128rr $xmm16 + $xmm16 = VMOVAPSZ128rr $xmm16 ; CHECK: VMOVDQA32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVDQA32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVDQA32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVDQA32Z128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVDQA32Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVDQA32Z128rr $xmm16 - $xmm16 = VMOVDQA32Z128rr $xmm16 + $xmm16 = VMOVDQA32Z128rr $xmm16 ; CHECK: VMOVDQA64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVDQA64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVDQA64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVDQA64Z128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVDQA64Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVDQA64Z128rr $xmm16 - $xmm16 = VMOVDQA64Z128rr $xmm16 + $xmm16 = VMOVDQA64Z128rr $xmm16 ; CHECK: VMOVDQU16Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVDQU16Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVDQU16Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVDQU16Z128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVDQU16Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVDQU16Z128rr $xmm16 - $xmm16 = VMOVDQU16Z128rr $xmm16 + $xmm16 = VMOVDQU16Z128rr $xmm16 ; CHECK: VMOVDQU32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVDQU32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVDQU32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVDQU32Z128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVDQU32Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVDQU32Z128rr $xmm16 - $xmm16 = VMOVDQU32Z128rr $xmm16 + $xmm16 = VMOVDQU32Z128rr $xmm16 ; CHECK: VMOVDQU64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVDQU64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVDQU64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVDQU64Z128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVDQU64Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVDQU64Z128rr $xmm16 - $xmm16 = VMOVDQU64Z128rr $xmm16 + $xmm16 = VMOVDQU64Z128rr $xmm16 ; CHECK: VMOVDQU8Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVDQU8Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVDQU8Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVDQU8Z128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVDQU8Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVDQU8Z128rr $xmm16 - $xmm16 = VMOVDQU8Z128rr $xmm16 + $xmm16 = VMOVDQU8Z128rr $xmm16 ; CHECK: $xmm16 = VMOVNTDQAZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVNTDQAZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: VMOVUPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVUPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVUPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVUPDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVUPDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVUPDZ128rr $xmm16 - $xmm16 = VMOVUPDZ128rr $xmm16 + $xmm16 = VMOVUPDZ128rr $xmm16 ; CHECK: VMOVUPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVUPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVUPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVUPSZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVUPSZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVUPSZ128rr $xmm16 - $xmm16 = VMOVUPSZ128rr $xmm16 + $xmm16 = VMOVUPSZ128rr $xmm16 ; CHECK: VMOVNTDQZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVNTDQZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVNTDQZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: VMOVNTPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVNTPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVNTPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: VMOVNTPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVNTPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVNTPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VPMOVSXBDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVSXBDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVSXBDZ128rr $xmm16 - $xmm16 = VPMOVSXBDZ128rr $xmm16 + $xmm16 = VPMOVSXBDZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVSXBQZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVSXBQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVSXBQZ128rr $xmm16 - $xmm16 = VPMOVSXBQZ128rr $xmm16 + $xmm16 = VPMOVSXBQZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVSXBWZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVSXBWZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVSXBWZ128rr $xmm16 - $xmm16 = VPMOVSXBWZ128rr $xmm16 + $xmm16 = VPMOVSXBWZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVSXDQZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVSXDQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVSXDQZ128rr $xmm16 - $xmm16 = VPMOVSXDQZ128rr $xmm16 + $xmm16 = VPMOVSXDQZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVSXWDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVSXWDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVSXWDZ128rr $xmm16 - $xmm16 = VPMOVSXWDZ128rr $xmm16 + $xmm16 = VPMOVSXWDZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVSXWQZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVSXWQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVSXWQZ128rr $xmm16 - $xmm16 = VPMOVSXWQZ128rr $xmm16 + $xmm16 = VPMOVSXWQZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVZXBDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVZXBDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVZXBDZ128rr $xmm16 - $xmm16 = VPMOVZXBDZ128rr $xmm16 + $xmm16 = VPMOVZXBDZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVZXBQZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVZXBQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVZXBQZ128rr $xmm16 - $xmm16 = VPMOVZXBQZ128rr $xmm16 + $xmm16 = VPMOVZXBQZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVZXBWZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVZXBWZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVZXBWZ128rr $xmm16 - $xmm16 = VPMOVZXBWZ128rr $xmm16 + $xmm16 = VPMOVZXBWZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVZXDQZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVZXDQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVZXDQZ128rr $xmm16 - $xmm16 = VPMOVZXDQZ128rr $xmm16 + $xmm16 = VPMOVZXDQZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVZXWDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVZXWDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVZXWDZ128rr $xmm16 - $xmm16 = VPMOVZXWDZ128rr $xmm16 + $xmm16 = VPMOVZXWDZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVZXWQZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVZXWQZ128rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPMOVZXWQZ128rr $xmm16 - $xmm16 = VPMOVZXWQZ128rr $xmm16 + ; CHECK: $xmm16 = VPMOVZXWQZ128rr $xmm16 + $xmm16 = VPMOVZXWQZ128rr $xmm16 ; CHECK: VMOVHPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVHPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVHPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVHPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VMOVHPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VMOVHPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: VMOVHPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVHPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVHPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVHPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VMOVHPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VMOVHPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: VMOVLPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVLPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVLPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVLPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VMOVLPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VMOVLPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: VMOVLPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVLPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - ; CHECK: $xmm16 = VMOVLPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VMOVLPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg + VMOVLPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + ; CHECK: $xmm16 = VMOVLPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VMOVLPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMAXCPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VMAXCPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VMAXCPDZ128rr $xmm16, $xmm1, implicit $mxcsr @@ -3403,183 +3403,183 @@ body: | ; CHECK: $xmm16 = VORPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VORPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VORPDZ128rr $xmm16, $xmm1 - $xmm16 = VORPDZ128rr $xmm16, $xmm1 + $xmm16 = VORPDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VORPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VORPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VORPSZ128rr $xmm16, $xmm1 - $xmm16 = VORPSZ128rr $xmm16, $xmm1 + $xmm16 = VORPSZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDBZ128rr $xmm16, $xmm1 - $xmm16 = VPADDBZ128rr $xmm16, $xmm1 + $xmm16 = VPADDBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDDZ128rr $xmm16, $xmm1 - $xmm16 = VPADDDZ128rr $xmm16, $xmm1 + $xmm16 = VPADDDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDQZ128rr $xmm16, $xmm1 - $xmm16 = VPADDQZ128rr $xmm16, $xmm1 + $xmm16 = VPADDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDSBZ128rr $xmm16, $xmm1 - $xmm16 = VPADDSBZ128rr $xmm16, $xmm1 + $xmm16 = VPADDSBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDSWZ128rr $xmm16, $xmm1 - $xmm16 = VPADDSWZ128rr $xmm16, $xmm1 + $xmm16 = VPADDSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDUSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDUSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDUSBZ128rr $xmm16, $xmm1 - $xmm16 = VPADDUSBZ128rr $xmm16, $xmm1 + $xmm16 = VPADDUSBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDUSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDUSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDUSWZ128rr $xmm16, $xmm1 - $xmm16 = VPADDUSWZ128rr $xmm16, $xmm1 + $xmm16 = VPADDUSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDWZ128rr $xmm16, $xmm1 - $xmm16 = VPADDWZ128rr $xmm16, $xmm1 + $xmm16 = VPADDWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPANDDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPANDDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPANDDZ128rr $xmm16, $xmm1 - $xmm16 = VPANDDZ128rr $xmm16, $xmm1 + $xmm16 = VPANDDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPANDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPANDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPANDQZ128rr $xmm16, $xmm1 - $xmm16 = VPANDQZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPANDQZ128rr $xmm16, $xmm1 + $xmm16 = VPANDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPANDNDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPANDNDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPANDNDZ128rr $xmm16, $xmm1 - $xmm16 = VPANDNDZ128rr $xmm16, $xmm1 + $xmm16 = VPANDNDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPANDNQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPANDNQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPANDNQZ128rr $xmm16, $xmm1 - $xmm16 = VPANDNQZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPANDNQZ128rr $xmm16, $xmm1 + $xmm16 = VPANDNQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPAVGBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPAVGBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPAVGBZ128rr $xmm16, $xmm1 - $xmm16 = VPAVGBZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPAVGBZ128rr $xmm16, $xmm1 + $xmm16 = VPAVGBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPAVGWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPAVGWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPAVGWZ128rr $xmm16, $xmm1 - $xmm16 = VPAVGWZ128rr $xmm16, $xmm1 + $xmm16 = VPAVGWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMAXSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMAXSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMAXSBZ128rr $xmm16, $xmm1 - $xmm16 = VPMAXSBZ128rr $xmm16, $xmm1 + $xmm16 = VPMAXSBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMAXSDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMAXSDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMAXSDZ128rr $xmm16, $xmm1 - $xmm16 = VPMAXSDZ128rr $xmm16, $xmm1 + $xmm16 = VPMAXSDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMAXSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMAXSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPMAXSWZ128rr $xmm16, $xmm1 - $xmm16 = VPMAXSWZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPMAXSWZ128rr $xmm16, $xmm1 + $xmm16 = VPMAXSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMAXUBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMAXUBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMAXUBZ128rr $xmm16, $xmm1 - $xmm16 = VPMAXUBZ128rr $xmm16, $xmm1 + $xmm16 = VPMAXUBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMAXUDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMAXUDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMAXUDZ128rr $xmm16, $xmm1 - $xmm16 = VPMAXUDZ128rr $xmm16, $xmm1 + $xmm16 = VPMAXUDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMAXUWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMAXUWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMAXUWZ128rr $xmm16, $xmm1 - $xmm16 = VPMAXUWZ128rr $xmm16, $xmm1 + $xmm16 = VPMAXUWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMINSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMINSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMINSBZ128rr $xmm16, $xmm1 - $xmm16 = VPMINSBZ128rr $xmm16, $xmm1 + $xmm16 = VPMINSBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMINSDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMINSDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMINSDZ128rr $xmm16, $xmm1 - $xmm16 = VPMINSDZ128rr $xmm16, $xmm1 + $xmm16 = VPMINSDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMINSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMINSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMINSWZ128rr $xmm16, $xmm1 - $xmm16 = VPMINSWZ128rr $xmm16, $xmm1 + $xmm16 = VPMINSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMINUBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMINUBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMINUBZ128rr $xmm16, $xmm1 - $xmm16 = VPMINUBZ128rr $xmm16, $xmm1 + $xmm16 = VPMINUBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMINUDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMINUDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMINUDZ128rr $xmm16, $xmm1 - $xmm16 = VPMINUDZ128rr $xmm16, $xmm1 + $xmm16 = VPMINUDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMINUWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMINUWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMINUWZ128rr $xmm16, $xmm1 - $xmm16 = VPMINUWZ128rr $xmm16, $xmm1 + $xmm16 = VPMINUWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMULDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMULDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMULDQZ128rr $xmm16, $xmm1 - $xmm16 = VPMULDQZ128rr $xmm16, $xmm1 + $xmm16 = VPMULDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMULHRSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMULHRSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMULHRSWZ128rr $xmm16, $xmm1 - $xmm16 = VPMULHRSWZ128rr $xmm16, $xmm1 + $xmm16 = VPMULHRSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMULHUWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMULHUWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMULHUWZ128rr $xmm16, $xmm1 - $xmm16 = VPMULHUWZ128rr $xmm16, $xmm1 + $xmm16 = VPMULHUWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMULHWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMULHWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMULHWZ128rr $xmm16, $xmm1 - $xmm16 = VPMULHWZ128rr $xmm16, $xmm1 + $xmm16 = VPMULHWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMULLDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMULLDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMULLDZ128rr $xmm16, $xmm1 - $xmm16 = VPMULLDZ128rr $xmm16, $xmm1 + $xmm16 = VPMULLDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMULLWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMULLWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMULLWZ128rr $xmm16, $xmm1 - $xmm16 = VPMULLWZ128rr $xmm16, $xmm1 + $xmm16 = VPMULLWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMULUDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMULUDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMULUDQZ128rr $xmm16, $xmm1 - $xmm16 = VPMULUDQZ128rr $xmm16, $xmm1 + $xmm16 = VPMULUDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPORDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPORDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPORDZ128rr $xmm16, $xmm1 - $xmm16 = VPORDZ128rr $xmm16, $xmm1 + $xmm16 = VPORDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPORQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPORQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPORQZ128rr $xmm16, $xmm1 - $xmm16 = VPORQZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPORQZ128rr $xmm16, $xmm1 + $xmm16 = VPORQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSUBBZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBBZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSUBDZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBDZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSUBQZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBQZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPSUBSBZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBSBZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPSUBSBZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBSBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSUBSWZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBSWZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBUSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBUSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPSUBUSBZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBUSBZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPSUBUSBZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBUSBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBUSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBUSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSUBUSWZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBUSWZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBUSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPSUBWZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBWZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPSUBWZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VADDPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VADDPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VADDPDZ128rr $xmm16, $xmm1, implicit $mxcsr @@ -3591,19 +3591,19 @@ body: | ; CHECK: $xmm16 = VANDNPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VANDNPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VANDNPDZ128rr $xmm16, $xmm1 - $xmm16 = VANDNPDZ128rr $xmm16, $xmm1 + $xmm16 = VANDNPDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VANDNPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VANDNPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VANDNPSZ128rr $xmm16, $xmm1 - $xmm16 = VANDNPSZ128rr $xmm16, $xmm1 + $xmm16 = VANDNPSZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VANDPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VANDPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VANDPDZ128rr $xmm16, $xmm1 - $xmm16 = VANDPDZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VANDPDZ128rr $xmm16, $xmm1 + $xmm16 = VANDPDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VANDPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VANDPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VANDPSZ128rr $xmm16, $xmm1 - $xmm16 = VANDPSZ128rr $xmm16, $xmm1 + $xmm16 = VANDPSZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VDIVPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VDIVPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VDIVPDZ128rr $xmm16, $xmm1, implicit $mxcsr @@ -3615,11 +3615,11 @@ body: | ; CHECK: $xmm16 = VPXORDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPXORDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPXORDZ128rr $xmm16, $xmm1 - $xmm16 = VPXORDZ128rr $xmm16, $xmm1 + $xmm16 = VPXORDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPXORQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPXORQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPXORQZ128rr $xmm16, $xmm1 - $xmm16 = VPXORQZ128rr $xmm16, $xmm1 + $xmm16 = VPXORQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VSUBPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VSUBPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VSUBPDZ128rr $xmm16, $xmm1, implicit $mxcsr @@ -3631,83 +3631,83 @@ body: | ; CHECK: $xmm16 = VXORPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VXORPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VXORPDZ128rr $xmm16, $xmm1 - $xmm16 = VXORPDZ128rr $xmm16, $xmm1 + $xmm16 = VXORPDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VXORPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VXORPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VXORPSZ128rr $xmm16, $xmm1 - $xmm16 = VXORPSZ128rr $xmm16, $xmm1 + $xmm16 = VXORPSZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMADDUBSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMADDUBSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMADDUBSWZ128rr $xmm16, $xmm1 - $xmm16 = VPMADDUBSWZ128rr $xmm16, $xmm1 + $xmm16 = VPMADDUBSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMADDWDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMADDWDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPMADDWDZ128rr $xmm16, $xmm1 - $xmm16 = VPMADDWDZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPMADDWDZ128rr $xmm16, $xmm1 + $xmm16 = VPMADDWDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPACKSSDWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPACKSSDWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPACKSSDWZ128rr $xmm16, $xmm1 - $xmm16 = VPACKSSDWZ128rr $xmm16, $xmm1 + $xmm16 = VPACKSSDWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPACKSSWBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPACKSSWBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPACKSSWBZ128rr $xmm16, $xmm1 - $xmm16 = VPACKSSWBZ128rr $xmm16, $xmm1 + $xmm16 = VPACKSSWBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPACKUSDWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPACKUSDWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPACKUSDWZ128rr $xmm16, $xmm1 - $xmm16 = VPACKUSDWZ128rr $xmm16, $xmm1 + $xmm16 = VPACKUSDWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPACKUSWBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPACKUSWBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPACKUSWBZ128rr $xmm16, $xmm1 - $xmm16 = VPACKUSWBZ128rr $xmm16, $xmm1 + $xmm16 = VPACKUSWBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKHBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKHBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKHBWZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKHBWZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKHBWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKHDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKHDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKHDQZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKHDQZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKHDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKHQDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKHQDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKHQDQZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKHQDQZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKHQDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKHWDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKHWDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKHWDZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKHWDZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKHWDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKLBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKLBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKLBWZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKLBWZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKLBWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKLDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKLDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKLDQZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKLDQZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKLDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKLQDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKLQDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKLQDQZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKLQDQZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKLQDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKLWDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKLWDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKLWDZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKLWDZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKLWDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VUNPCKHPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VUNPCKHPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VUNPCKHPDZ128rr $xmm16, $xmm1 - $xmm16 = VUNPCKHPDZ128rr $xmm16, $xmm1 + $xmm16 = VUNPCKHPDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VUNPCKHPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VUNPCKHPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VUNPCKHPSZ128rr $xmm16, $xmm1 - $xmm16 = VUNPCKHPSZ128rr $xmm16, $xmm1 + $xmm16 = VUNPCKHPSZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VUNPCKLPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VUNPCKLPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VUNPCKLPDZ128rr $xmm16, $xmm1 - $xmm16 = VUNPCKLPDZ128rr $xmm16, $xmm1 + $xmm16 = VUNPCKLPDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VUNPCKLPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VUNPCKLPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VUNPCKLPSZ128rr $xmm16, $xmm1 - $xmm16 = VUNPCKLPSZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VUNPCKLPSZ128rr $xmm16, $xmm1 + $xmm16 = VUNPCKLPSZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VFMADD132PDZ128m $xmm16, $xmm16, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VFMADD132PDZ128m $xmm16, $xmm16, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VFMADD132PDZ128r $xmm16, $xmm1, $xmm2, implicit $mxcsr @@ -3852,14 +3852,14 @@ body: | $xmm16 = VFNMSUB231PSZ128m $xmm16, $xmm16, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VFNMSUB231PSZ128r $xmm16, $xmm1, $xmm2, implicit $mxcsr $xmm16 = VFNMSUB231PSZ128r $xmm16, $xmm1, $xmm2, implicit $mxcsr - ; CHECK: $xmm16 = VPSLLDZ128ri $xmm16, 7 - $xmm16 = VPSLLDZ128ri $xmm16, 7 + ; CHECK: $xmm16 = VPSLLDZ128ri $xmm16, 7 + $xmm16 = VPSLLDZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSLLDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSLLDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSLLDZ128rr $xmm16, $xmm16 $xmm16 = VPSLLDZ128rr $xmm16, $xmm16 ; CHECK: $xmm16 = VPSLLQZ128ri $xmm16, 7 - $xmm16 = VPSLLQZ128ri $xmm16, 7 + $xmm16 = VPSLLQZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSLLQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSLLQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSLLQZ128rr $xmm16, $xmm16 @@ -3873,13 +3873,13 @@ body: | ; CHECK: $xmm16 = VPSLLVQZ128rr $xmm16, $xmm16 $xmm16 = VPSLLVQZ128rr $xmm16, $xmm16 ; CHECK: $xmm16 = VPSLLWZ128ri $xmm16, 7 - $xmm16 = VPSLLWZ128ri $xmm16, 7 + $xmm16 = VPSLLWZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSLLWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSLLWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSLLWZ128rr $xmm16, $xmm16 $xmm16 = VPSLLWZ128rr $xmm16, $xmm16 ; CHECK: $xmm16 = VPSRADZ128ri $xmm16, 7 - $xmm16 = VPSRADZ128ri $xmm16, 7 + $xmm16 = VPSRADZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSRADZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSRADZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSRADZ128rr $xmm16, $xmm16 @@ -3888,22 +3888,22 @@ body: | $xmm16 = VPSRAVDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSRAVDZ128rr $xmm16, $xmm16 $xmm16 = VPSRAVDZ128rr $xmm16, $xmm16 - ; CHECK: $xmm16 = VPSRAWZ128ri $xmm16, 7 - $xmm16 = VPSRAWZ128ri $xmm16, 7 + ; CHECK: $xmm16 = VPSRAWZ128ri $xmm16, 7 + $xmm16 = VPSRAWZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSRAWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSRAWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSRAWZ128rr $xmm16, $xmm16 $xmm16 = VPSRAWZ128rr $xmm16, $xmm16 ; CHECK: $xmm16 = VPSRLDQZ128ri $xmm16, 14 - $xmm16 = VPSRLDQZ128ri $xmm16, 14 - ; CHECK: $xmm16 = VPSRLDZ128ri $xmm16, 7 - $xmm16 = VPSRLDZ128ri $xmm16, 7 + $xmm16 = VPSRLDQZ128ri $xmm16, 14 + ; CHECK: $xmm16 = VPSRLDZ128ri $xmm16, 7 + $xmm16 = VPSRLDZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSRLDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSRLDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSRLDZ128rr $xmm16, $xmm16 $xmm16 = VPSRLDZ128rr $xmm16, $xmm16 - ; CHECK: $xmm16 = VPSRLQZ128ri $xmm16, 7 - $xmm16 = VPSRLQZ128ri $xmm16, 7 + ; CHECK: $xmm16 = VPSRLQZ128ri $xmm16, 7 + $xmm16 = VPSRLQZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSRLQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSRLQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSRLQZ128rr $xmm16, $xmm16 @@ -3917,7 +3917,7 @@ body: | ; CHECK: $xmm16 = VPSRLVQZ128rr $xmm16, $xmm16 $xmm16 = VPSRLVQZ128rr $xmm16, $xmm16 ; CHECK: $xmm16 = VPSRLWZ128ri $xmm16, 7 - $xmm16 = VPSRLWZ128ri $xmm16, 7 + $xmm16 = VPSRLWZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSRLWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSRLWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSRLWZ128rr $xmm16, $xmm16 @@ -3925,27 +3925,27 @@ body: | ; CHECK: $xmm16 = VPERMILPDZ128mi $rdi, 1, $noreg, 0, $noreg, 9 $xmm16 = VPERMILPDZ128mi $rdi, 1, $noreg, 0, $noreg, 9 ; CHECK: $xmm16 = VPERMILPDZ128ri $xmm16, 9 - $xmm16 = VPERMILPDZ128ri $xmm16, 9 + $xmm16 = VPERMILPDZ128ri $xmm16, 9 ; CHECK: $xmm16 = VPERMILPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VPERMILPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VPERMILPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPERMILPDZ128rr $xmm16, $xmm1 - $xmm16 = VPERMILPDZ128rr $xmm16, $xmm1 + $xmm16 = VPERMILPDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPERMILPSZ128mi $rdi, 1, $noreg, 0, $noreg, 9 $xmm16 = VPERMILPSZ128mi $rdi, 1, $noreg, 0, $noreg, 9 ; CHECK: $xmm16 = VPERMILPSZ128ri $xmm16, 9 - $xmm16 = VPERMILPSZ128ri $xmm16, 9 + $xmm16 = VPERMILPSZ128ri $xmm16, 9 ; CHECK: $xmm16 = VPERMILPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VPERMILPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VPERMILPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPERMILPSZ128rr $xmm16, $xmm1 - $xmm16 = VPERMILPSZ128rr $xmm16, $xmm1 + $xmm16 = VPERMILPSZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VCVTPH2PSZ128rm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VCVTPH2PSZ128rm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VCVTPH2PSZ128rr $xmm16, implicit $mxcsr $xmm16 = VCVTPH2PSZ128rr $xmm16, implicit $mxcsr ; CHECK: $xmm16 = VCVTDQ2PDZ128rm $rdi, 1, $noreg, 0, $noreg $xmm16 = VCVTDQ2PDZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VCVTDQ2PDZ128rr $xmm16 - $xmm16 = VCVTDQ2PDZ128rr $xmm16 + ; CHECK: $xmm16 = VCVTDQ2PDZ128rr $xmm16 + $xmm16 = VCVTDQ2PDZ128rr $xmm16 ; CHECK: $xmm16 = VCVTDQ2PSZ128rm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VCVTDQ2PSZ128rm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VCVTDQ2PSZ128rr $xmm16, implicit $mxcsr @@ -3982,34 +3982,34 @@ body: | $xmm16 = VSQRTPSZ128m $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VSQRTPSZ128r $xmm16, implicit $mxcsr $xmm16 = VSQRTPSZ128r $xmm16, implicit $mxcsr - ; CHECK: $xmm16 = VMOVDDUPZ128rm $rdi, 1, $noreg, 0, $noreg - $xmm16 = VMOVDDUPZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VMOVDDUPZ128rr $xmm16 - $xmm16 = VMOVDDUPZ128rr $xmm16 - ; CHECK: $xmm16 = VMOVSHDUPZ128rm $rdi, 1, $noreg, 0, $noreg - $xmm16 = VMOVSHDUPZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VMOVSHDUPZ128rr $xmm16 - $xmm16 = VMOVSHDUPZ128rr $xmm16 - ; CHECK: $xmm16 = VMOVSLDUPZ128rm $rdi, 1, $noreg, 0, $noreg - $xmm16 = VMOVSLDUPZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VMOVSLDUPZ128rr $xmm16 - $xmm16 = VMOVSLDUPZ128rr $xmm16 + ; CHECK: $xmm16 = VMOVDDUPZ128rm $rdi, 1, $noreg, 0, $noreg + $xmm16 = VMOVDDUPZ128rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $xmm16 = VMOVDDUPZ128rr $xmm16 + $xmm16 = VMOVDDUPZ128rr $xmm16 + ; CHECK: $xmm16 = VMOVSHDUPZ128rm $rdi, 1, $noreg, 0, $noreg + $xmm16 = VMOVSHDUPZ128rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $xmm16 = VMOVSHDUPZ128rr $xmm16 + $xmm16 = VMOVSHDUPZ128rr $xmm16 + ; CHECK: $xmm16 = VMOVSLDUPZ128rm $rdi, 1, $noreg, 0, $noreg + $xmm16 = VMOVSLDUPZ128rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $xmm16 = VMOVSLDUPZ128rr $xmm16 + $xmm16 = VMOVSLDUPZ128rr $xmm16 ; CHECK: $xmm16 = VPSHUFBZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg $xmm16 = VPSHUFBZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSHUFBZ128rr $xmm16, $xmm1 - $xmm16 = VPSHUFBZ128rr $xmm16, $xmm1 + $xmm16 = VPSHUFBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSHUFDZ128mi $rdi, 1, $noreg, 0, $noreg, -24 $xmm16 = VPSHUFDZ128mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $xmm16 = VPSHUFDZ128ri $xmm16, -24 - $xmm16 = VPSHUFDZ128ri $xmm16, -24 + $xmm16 = VPSHUFDZ128ri $xmm16, -24 ; CHECK: $xmm16 = VPSHUFHWZ128mi $rdi, 1, $noreg, 0, $noreg, -24 $xmm16 = VPSHUFHWZ128mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $xmm16 = VPSHUFHWZ128ri $xmm16, -24 - $xmm16 = VPSHUFHWZ128ri $xmm16, -24 + $xmm16 = VPSHUFHWZ128ri $xmm16, -24 ; CHECK: $xmm16 = VPSHUFLWZ128mi $rdi, 1, $noreg, 0, $noreg, -24 $xmm16 = VPSHUFLWZ128mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $xmm16 = VPSHUFLWZ128ri $xmm16, -24 - $xmm16 = VPSHUFLWZ128ri $xmm16, -24 + $xmm16 = VPSHUFLWZ128ri $xmm16, -24 ; CHECK: $xmm16 = VPSLLDQZ128ri $xmm16, 1 $xmm16 = VPSLLDQZ128ri $xmm16, 1 ; CHECK: $xmm16 = VSHUFPDZ128rmi $xmm16, $rip, 1, $noreg, 0, $noreg, -24 @@ -4022,28 +4022,28 @@ body: | $xmm16 = VSHUFPSZ128rri $xmm16, $xmm1, -24 ; CHECK: $xmm16 = VPSADBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSADBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPSADBWZ128rr $xmm16, $xmm1 - $xmm16 = VPSADBWZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPSADBWZ128rr $xmm16, $xmm1 + $xmm16 = VPSADBWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VBROADCASTSSZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VBROADCASTSSZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VBROADCASTSSZ128rr $xmm16 - $xmm16 = VBROADCASTSSZ128rr $xmm16 + $xmm16 = VBROADCASTSSZ128rr $xmm16 ; CHECK: $xmm16 = VPBROADCASTBZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPBROADCASTBZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPBROADCASTBZ128rr $xmm16 - $xmm16 = VPBROADCASTBZ128rr $xmm16 + $xmm16 = VPBROADCASTBZ128rr $xmm16 ; CHECK: $xmm16 = VPBROADCASTDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPBROADCASTDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPBROADCASTDZ128rr $xmm16 - $xmm16 = VPBROADCASTDZ128rr $xmm16 + $xmm16 = VPBROADCASTDZ128rr $xmm16 ; CHECK: $xmm16 = VPBROADCASTQZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPBROADCASTQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPBROADCASTQZ128rr $xmm16 - $xmm16 = VPBROADCASTQZ128rr $xmm16 + $xmm16 = VPBROADCASTQZ128rr $xmm16 ; CHECK: $xmm16 = VPBROADCASTWZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPBROADCASTWZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPBROADCASTWZ128rr $xmm16 - $xmm16 = VPBROADCASTWZ128rr $xmm16 + $xmm16 = VPBROADCASTWZ128rr $xmm16 ; CHECK: $xmm16 = VBROADCASTI32X2Z128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VBROADCASTI32X2Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VBROADCASTI32X2Z128rr $xmm0 @@ -4055,15 +4055,15 @@ body: | ; CHECK: $xmm16 = VPABSBZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPABSBZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPABSBZ128rr $xmm16 - $xmm16 = VPABSBZ128rr $xmm16 + $xmm16 = VPABSBZ128rr $xmm16 ; CHECK: $xmm16 = VPABSDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPABSDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPABSDZ128rr $xmm16 - $xmm16 = VPABSDZ128rr $xmm16 + $xmm16 = VPABSDZ128rr $xmm16 ; CHECK: $xmm16 = VPABSWZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPABSWZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPABSWZ128rr $xmm16 - $xmm16 = VPABSWZ128rr $xmm16 + $xmm16 = VPABSWZ128rr $xmm16 ; CHECK: $xmm16 = VPALIGNRZ128rmi $xmm16, $rdi, 1, $noreg, 0, $noreg, 15 $xmm16 = VPALIGNRZ128rmi $xmm16, $rdi, 1, $noreg, 0, $noreg, 15 ; CHECK: $xmm16 = VPALIGNRZ128rri $xmm16, $xmm1, 15 @@ -4092,12 +4092,12 @@ body: | $xmm0 = VRNDSCALEPSZ128rmi $rip, 1, $noreg, 0, $noreg, 31, implicit $mxcsr ; CHECK: $xmm0 = VRNDSCALEPSZ128rri $xmm0, 31, implicit $mxcsr $xmm0 = VRNDSCALEPSZ128rri $xmm0, 31, implicit $mxcsr - + RET64 ... --- # CHECK-LABEL: name: evex_scalar_to_evex_test - # CHECK: bb.0: + # CHECK: bb.0: name: evex_scalar_to_evex_test body: | @@ -4406,38 +4406,38 @@ body: | $xmm16 = VFNMSUB231SSZr $xmm16, $xmm1, $xmm2, implicit $mxcsr ; CHECK: $xmm16 = VFNMSUB231SSZr_Int $xmm16, $xmm1, $xmm2, implicit $mxcsr $xmm16 = VFNMSUB231SSZr_Int $xmm16, $xmm1, $xmm2, implicit $mxcsr - ; CHECK: VPEXTRBZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - VPEXTRBZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - ; CHECK: $eax = VPEXTRBZrr $xmm16, 1 - $eax = VPEXTRBZrr $xmm16, 1 - ; CHECK: VPEXTRDZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - VPEXTRDZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - ; CHECK: $eax = VPEXTRDZrr $xmm16, 1 - $eax = VPEXTRDZrr $xmm16, 1 - ; CHECK: VPEXTRQZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - VPEXTRQZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - ; CHECK: $rax = VPEXTRQZrr $xmm16, 1 - $rax = VPEXTRQZrr $xmm16, 1 - ; CHECK: VPEXTRWZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - VPEXTRWZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - ; CHECK: $eax = VPEXTRWZrr $xmm16, 1 - $eax = VPEXTRWZrr $xmm16, 1 - ; CHECK: $xmm16 = VPINSRBZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm16 = VPINSRBZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 - ; CHECK: $xmm16 = VPINSRBZrr $xmm16, $edi, 5 - $xmm16 = VPINSRBZrr $xmm16, $edi, 5 - ; CHECK: $xmm16 = VPINSRDZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm16 = VPINSRDZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 - ; CHECK: $xmm16 = VPINSRDZrr $xmm16, $edi, 5 - $xmm16 = VPINSRDZrr $xmm16, $edi, 5 - ; CHECK: $xmm16 = VPINSRQZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm16 = VPINSRQZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 - ; CHECK: $xmm16 = VPINSRQZrr $xmm16, $rdi, 5 - $xmm16 = VPINSRQZrr $xmm16, $rdi, 5 - ; CHECK: $xmm16 = VPINSRWZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm16 = VPINSRWZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: VPEXTRBZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + VPEXTRBZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + ; CHECK: $eax = VPEXTRBZrr $xmm16, 1 + $eax = VPEXTRBZrr $xmm16, 1 + ; CHECK: VPEXTRDZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + VPEXTRDZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + ; CHECK: $eax = VPEXTRDZrr $xmm16, 1 + $eax = VPEXTRDZrr $xmm16, 1 + ; CHECK: VPEXTRQZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + VPEXTRQZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + ; CHECK: $rax = VPEXTRQZrr $xmm16, 1 + $rax = VPEXTRQZrr $xmm16, 1 + ; CHECK: VPEXTRWZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + VPEXTRWZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + ; CHECK: $eax = VPEXTRWZrr $xmm16, 1 + $eax = VPEXTRWZrr $xmm16, 1 + ; CHECK: $xmm16 = VPINSRBZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm16 = VPINSRBZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: $xmm16 = VPINSRBZrr $xmm16, $edi, 5 + $xmm16 = VPINSRBZrr $xmm16, $edi, 5 + ; CHECK: $xmm16 = VPINSRDZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm16 = VPINSRDZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: $xmm16 = VPINSRDZrr $xmm16, $edi, 5 + $xmm16 = VPINSRDZrr $xmm16, $edi, 5 + ; CHECK: $xmm16 = VPINSRQZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm16 = VPINSRQZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: $xmm16 = VPINSRQZrr $xmm16, $rdi, 5 + $xmm16 = VPINSRQZrr $xmm16, $rdi, 5 + ; CHECK: $xmm16 = VPINSRWZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm16 = VPINSRWZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 ; CHECK: $xmm16 = VPINSRWZrr $xmm16, $edi, 5 - $xmm16 = VPINSRWZrr $xmm16, $edi, 5 + $xmm16 = VPINSRWZrr $xmm16, $edi, 5 ; CHECK: $xmm16 = VSQRTSDZm $xmm16, $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VSQRTSDZm $xmm16, $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VSQRTSDZm_Int $xmm16, $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr @@ -4471,9 +4471,9 @@ body: | ; CHECK: $xmm16 = VCVTSD2SSZrr_Int $xmm16, $xmm16, implicit $mxcsr $xmm16 = VCVTSD2SSZrr_Int $xmm16, $xmm16, implicit $mxcsr ; CHECK: $xmm16 = VCVTSI2SDZrm $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VCVTSI2SDZrm $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VCVTSI2SDZrm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VCVTSI2SDZrm_Int $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VCVTSI2SDZrm_Int $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VCVTSI2SDZrm_Int $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VCVTSI2SDZrr $xmm16, $edi $xmm16 = VCVTSI2SDZrr $xmm16, $edi ; CHECK: $xmm16 = VCVTSI2SDZrr_Int $xmm16, $edi @@ -4550,10 +4550,10 @@ body: | $edi = VCVTTSS2SIZrr $xmm16, implicit $mxcsr ; CHECK: $edi = VCVTTSS2SIZrr_Int $xmm16, implicit $mxcsr $edi = VCVTTSS2SIZrr_Int $xmm16, implicit $mxcsr - ; CHECK: $xmm16 = VMOV64toSDZrr $rdi - $xmm16 = VMOV64toSDZrr $rdi + ; CHECK: $xmm16 = VMOV64toSDZrr $rdi + $xmm16 = VMOV64toSDZrr $rdi ; CHECK: $xmm16 = VMOVDI2SSZrr $eax - $xmm16 = VMOVDI2SSZrr $eax + $xmm16 = VMOVDI2SSZrr $eax ; CHECK: VMOVSDZmr $rdi, 1, $noreg, 0, $noreg, $xmm16 VMOVSDZmr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVSDZrm $rip, 1, $noreg, 0, $noreg @@ -4561,7 +4561,7 @@ body: | ; CHECK: $xmm16 = VMOVSDZrm_alt $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVSDZrm_alt $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVSDZrr $xmm16, $xmm1 - $xmm16 = VMOVSDZrr $xmm16, $xmm1 + $xmm16 = VMOVSDZrr $xmm16, $xmm1 ; CHECK: $rax = VMOVSDto64Zrr $xmm16 $rax = VMOVSDto64Zrr $xmm16 ; CHECK: VMOVSSZmr $rdi, 1, $noreg, 0, $noreg, $xmm16 @@ -4571,19 +4571,19 @@ body: | ; CHECK: $xmm16 = VMOVSSZrm_alt $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVSSZrm_alt $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVSSZrr $xmm16, $xmm1 - $xmm16 = VMOVSSZrr $xmm16, $xmm1 + $xmm16 = VMOVSSZrr $xmm16, $xmm1 ; CHECK: $eax = VMOVSS2DIZrr $xmm16 $eax = VMOVSS2DIZrr $xmm16 ; CHECK: $xmm16 = VMOV64toPQIZrr $rdi - $xmm16 = VMOV64toPQIZrr $rdi + $xmm16 = VMOV64toPQIZrr $rdi ; CHECK: $xmm16 = VMOV64toPQIZrm $rdi, 1, $noreg, 0, $noreg $xmm16 = VMOV64toPQIZrm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VMOV64toSDZrr $rdi - $xmm16 = VMOV64toSDZrr $rdi + ; CHECK: $xmm16 = VMOV64toSDZrr $rdi + $xmm16 = VMOV64toSDZrr $rdi ; CHECK: $xmm16 = VMOVDI2PDIZrm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVDI2PDIZrm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVDI2PDIZrr $edi - $xmm16 = VMOVDI2PDIZrr $edi + $xmm16 = VMOVDI2PDIZrr $edi ; CHECK: $xmm16 = VMOVLHPSZrr $xmm16, $xmm1 $xmm16 = VMOVLHPSZrr $xmm16, $xmm1 ; CHECK: $xmm16 = VMOVHLPSZrr $xmm16, $xmm1 @@ -4591,19 +4591,19 @@ body: | ; CHECK: VMOVPDI2DIZmr $rdi, 1, $noreg, 0, $noreg, $xmm16 VMOVPDI2DIZmr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $edi = VMOVPDI2DIZrr $xmm16 - $edi = VMOVPDI2DIZrr $xmm16 + $edi = VMOVPDI2DIZrr $xmm16 ; CHECK: $xmm16 = VMOVPQI2QIZrr $xmm16 $xmm16 = VMOVPQI2QIZrr $xmm16 ; CHECK: VMOVPQI2QIZmr $rdi, 1, $noreg, 0, $noreg, $xmm16 VMOVPQI2QIZmr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $rdi = VMOVPQIto64Zrr $xmm16 - $rdi = VMOVPQIto64Zrr $xmm16 + $rdi = VMOVPQIto64Zrr $xmm16 ; CHECK: VMOVPQIto64Zmr $rdi, 1, $noreg, 0, $noreg, $xmm16 VMOVPQIto64Zmr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVQI2PQIZrm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVQI2PQIZrm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVZPQILo2PQIZrr $xmm16 - $xmm16 = VMOVZPQILo2PQIZrr $xmm16 + $xmm16 = VMOVZPQILo2PQIZrr $xmm16 ; CHECK: VCOMISDZrm_Int $xmm16, $rdi, 1, $noreg, 0, $noreg, implicit-def $eflags, implicit $mxcsr VCOMISDZrm_Int $xmm16, $rdi, 1, $noreg, 0, $noreg, implicit-def $eflags, implicit $mxcsr ; CHECK: VCOMISDZrr_Int $xmm16, $xmm1, implicit-def $eflags, implicit $mxcsr @@ -4668,6 +4668,6 @@ body: | $xmm0 = VRNDSCALESSZm_Int $xmm0, $rip, 1, $noreg, 0, $noreg, 31, implicit $mxcsr ; CHECK: $xmm0 = VRNDSCALESSZr_Int $xmm0, $xmm1, 31, implicit $mxcsr $xmm0 = VRNDSCALESSZr_Int $xmm0, $xmm1, 31, implicit $mxcsr - + RET64 ... diff --git a/llvm/test/CodeGen/X86/fold-load-vec.ll b/llvm/test/CodeGen/X86/fold-load-vec.ll index 348929cdf9f79e4ffc118edfb975f75227ab2437..0bf846a0930bb4c6a9440a27b84587a4b049f43e 100644 --- a/llvm/test/CodeGen/X86/fold-load-vec.ll +++ b/llvm/test/CodeGen/X86/fold-load-vec.ll @@ -10,8 +10,8 @@ define void @sample_test(ptr %source, ptr %dest) nounwind { ; CHECK-NEXT: subq $24, %rsp ; CHECK-NEXT: movq %rdi, {{[0-9]+}}(%rsp) ; CHECK-NEXT: movq %rsi, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movq $0, (%rsp) ; CHECK-NEXT: xorps %xmm0, %xmm0 -; CHECK-NEXT: movlps %xmm0, (%rsp) ; CHECK-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1] ; CHECK-NEXT: movlps %xmm0, (%rsp) ; CHECK-NEXT: movlps %xmm0, (%rsi) diff --git a/llvm/test/CodeGen/X86/fold-pcmpeqd-2.ll b/llvm/test/CodeGen/X86/fold-pcmpeqd-2.ll index 5dcb1d63207d149e2dd04b9f0e0291fd13984705..493e7aa7039e3fc7e5eeb0481f049dc4ac01ca40 100644 --- a/llvm/test/CodeGen/X86/fold-pcmpeqd-2.ll +++ b/llvm/test/CodeGen/X86/fold-pcmpeqd-2.ll @@ -51,11 +51,6 @@ define void @program_1(ptr %dest, ptr %t0, <4 x float> %p0, <4 x float> %p1, <4 ; X86-NEXT: movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload ; X86-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 ; X86-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 16-byte Spill -; X86-NEXT: xorps %xmm0, %xmm0 -; X86-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 16-byte Spill -; X86-NEXT: movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload -; X86-NEXT: mulps %xmm0, %xmm0 -; X86-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 16-byte Spill ; X86-NEXT: movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload ; X86-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 ; X86-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 16-byte Spill @@ -64,8 +59,10 @@ define void @program_1(ptr %dest, ptr %t0, <4 x float> %p0, <4 x float> %p1, <4 ; X86-NEXT: movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload ; X86-NEXT: cmpunordps %xmm0, %xmm0 ; X86-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 16-byte Spill +; X86-NEXT: xorps %xmm0, %xmm0 +; X86-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 16-byte Spill ; X86-NEXT: movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload -; X86-NEXT: minps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 +; X86-NEXT: minps %xmm0, %xmm0 ; X86-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 16-byte Spill ; X86-NEXT: xorps %xmm0, %xmm0 ; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp) @@ -135,11 +132,6 @@ define void @program_1(ptr %dest, ptr %t0, <4 x float> %p0, <4 x float> %p1, <4 ; X64-NEXT: movaps (%rsp), %xmm0 ## 16-byte Reload ; X64-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; X64-NEXT: movaps %xmm0, (%rsp) ## 16-byte Spill -; X64-NEXT: xorps %xmm0, %xmm0 -; X64-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill -; X64-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload -; X64-NEXT: mulps %xmm0, %xmm0 -; X64-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill ; X64-NEXT: movaps (%rsp), %xmm0 ## 16-byte Reload ; X64-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; X64-NEXT: movaps %xmm0, (%rsp) ## 16-byte Spill @@ -148,8 +140,10 @@ define void @program_1(ptr %dest, ptr %t0, <4 x float> %p0, <4 x float> %p1, <4 ; X64-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload ; X64-NEXT: cmpunordps %xmm0, %xmm0 ; X64-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill +; X64-NEXT: xorps %xmm0, %xmm0 +; X64-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill ; X64-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload -; X64-NEXT: minps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: minps %xmm0, %xmm0 ; X64-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill ; X64-NEXT: xorl %ebx, %ebx ; X64-NEXT: xorps %xmm3, %xmm3 diff --git a/llvm/test/CodeGen/X86/fp-intrinsics.ll b/llvm/test/CodeGen/X86/fp-intrinsics.ll index 32e45adcb94d71f5b6976e3dd79789c76dc35c24..5f77e2cb46cbf06c72ec94e9ddd05fba0113d34f 100644 --- a/llvm/test/CodeGen/X86/fp-intrinsics.ll +++ b/llvm/test/CodeGen/X86/fp-intrinsics.ll @@ -865,9 +865,9 @@ define double @f19() #0 { ; X87-NEXT: .cfi_def_cfa_offset 32 ; X87-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} ; X87-NEXT: fstpl {{[0-9]+}}(%esp) +; X87-NEXT: fld1 +; X87-NEXT: fstpl (%esp) ; X87-NEXT: wait -; X87-NEXT: movl $1072693248, {{[0-9]+}}(%esp) # imm = 0x3FF00000 -; X87-NEXT: movl $0, (%esp) ; X87-NEXT: calll fmod ; X87-NEXT: addl $28, %esp ; X87-NEXT: .cfi_def_cfa_offset 4 diff --git a/llvm/test/CodeGen/X86/gep-expanded-vector.ll b/llvm/test/CodeGen/X86/gep-expanded-vector.ll index 861de9c90085d67503a1e7b0ef322aa35c10e984..943cd3610c9d3243cc925d550fd93abda3cd0a00 100644 --- a/llvm/test/CodeGen/X86/gep-expanded-vector.ll +++ b/llvm/test/CodeGen/X86/gep-expanded-vector.ll @@ -1,24 +1,42 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc < %s -O2 -mattr=avx512f -mtriple=x86_64-unknown | FileCheck %s %struct.S1 = type { ptr, ptr } -define ptr @malloc_init_state(<64 x ptr> %tmp, i32 %ind) { +define ptr @malloc_init_state(<64 x ptr> %tmp, i32 %ind) nounwind { +; CHECK-LABEL: malloc_init_state: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: movq %rsp, %rbp +; CHECK-NEXT: andq $-64, %rsp +; CHECK-NEXT: subq $576, %rsp # imm = 0x240 +; CHECK-NEXT: # kill: def $edi killed $edi def $rdi +; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm8 = [16,16,16,16,16,16,16,16] +; CHECK-NEXT: vpaddq %zmm8, %zmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm8, %zmm1, %zmm1 +; CHECK-NEXT: vpaddq %zmm8, %zmm2, %zmm2 +; CHECK-NEXT: vpaddq %zmm8, %zmm3, %zmm3 +; CHECK-NEXT: vpaddq %zmm8, %zmm4, %zmm4 +; CHECK-NEXT: vpaddq %zmm8, %zmm5, %zmm5 +; CHECK-NEXT: vpaddq %zmm8, %zmm6, %zmm6 +; CHECK-NEXT: vpaddq %zmm8, %zmm7, %zmm7 +; CHECK-NEXT: vmovdqa64 %zmm7, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovdqa64 %zmm6, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovdqa64 %zmm5, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovdqa64 %zmm4, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovdqa64 %zmm3, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovdqa64 %zmm2, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovdqa64 %zmm1, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovdqa64 %zmm0, (%rsp) +; CHECK-NEXT: andl $63, %edi +; CHECK-NEXT: movq (%rsp,%rdi,8), %rax +; CHECK-NEXT: movq %rbp, %rsp +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq entry: %Vec = getelementptr inbounds ptr, <64 x ptr> %tmp , i64 2 %ptr = extractelement <64 x ptr> %Vec, i32 %ind ret ptr %ptr } -; CHECK: .LCPI0_0: -; CHECK: .quad 16 -; CHECK: vpbroadcastq .LCPI0_0(%rip), [[Z1:%zmm[0-9]]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z2:%zmm[0-9]]], [[Z2]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z3:%zmm[0-9]]], [[Z3]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z4:%zmm[0-9]]], [[Z4]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z5:%zmm[0-9]]], [[Z5]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z6:%zmm[0-9]]], [[Z6]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z7:%zmm[0-9]]], [[Z7]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z8:%zmm[0-9]]], [[Z8]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z9:%zmm[0-9]]], [[Z9]] - - diff --git a/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll b/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll index 6cd85e074c648e0543ca5f48457030a0a6ca10e9..8f1272013c11b6c0cb0b1cc0d3690689c475ab7f 100644 --- a/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll +++ b/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll @@ -31,7 +31,7 @@ define <16 x i8> @splatconstant_fshl_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $3, %xmm0, %xmm2 ; GFNIAVX512-NEXT: vpsrlw $5, %xmm1, %xmm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm2, %xmm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0 ; GFNIAVX512-NEXT: retq %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> ) ret <16 x i8> %res @@ -119,7 +119,7 @@ define <32 x i8> @splatconstant_fshl_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $4, %ymm0, %ymm2 ; GFNIAVX512-NEXT: vpsrlw $4, %ymm1, %ymm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; GFNIAVX512-NEXT: retq %res = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> ) ret <32 x i8> %res @@ -175,7 +175,7 @@ define <32 x i8> @splatconstant_fshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $2, %ymm0, %ymm2 ; GFNIAVX512-NEXT: vpsrlw $6, %ymm1, %ymm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; GFNIAVX512-NEXT: retq %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> ) ret <32 x i8> %res @@ -339,7 +339,7 @@ define <64 x i8> @splatconstant_fshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $6, %zmm0, %zmm2 ; GFNIAVX512-NEXT: vpsrlw $2, %zmm1, %zmm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; GFNIAVX512-NEXT: retq %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> ) ret <64 x i8> %res diff --git a/llvm/test/CodeGen/X86/gfni-rotates.ll b/llvm/test/CodeGen/X86/gfni-rotates.ll index 29b58d047596d40becde323e4b6caa40aa5a64bf..8b94a3f84cb96021455bda15596b85201bdb12d2 100644 --- a/llvm/test/CodeGen/X86/gfni-rotates.ll +++ b/llvm/test/CodeGen/X86/gfni-rotates.ll @@ -32,7 +32,7 @@ define <16 x i8> @splatconstant_rotl_v16i8(<16 x i8> %a) nounwind { ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $3, %xmm0, %xmm1 ; GFNIAVX512-NEXT: vpsrlw $5, %xmm0, %xmm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; GFNIAVX512-NEXT: retq %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %a, <16 x i8> ) ret <16 x i8> %res @@ -121,7 +121,7 @@ define <32 x i8> @splatconstant_rotl_v32i8(<32 x i8> %a) nounwind { ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $4, %ymm0, %ymm1 ; GFNIAVX512-NEXT: vpsrlw $4, %ymm0, %ymm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; GFNIAVX512-NEXT: retq %res = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a, <32 x i8> %a, <32 x i8> ) ret <32 x i8> %res @@ -177,7 +177,7 @@ define <32 x i8> @splatconstant_rotr_v32i8(<32 x i8> %a) nounwind { ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $2, %ymm0, %ymm1 ; GFNIAVX512-NEXT: vpsrlw $6, %ymm0, %ymm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; GFNIAVX512-NEXT: retq %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a, <32 x i8> %a, <32 x i8> ) ret <32 x i8> %res @@ -344,7 +344,7 @@ define <64 x i8> @splatconstant_rotr_v64i8(<64 x i8> %a) nounwind { ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $6, %zmm0, %zmm1 ; GFNIAVX512-NEXT: vpsrlw $2, %zmm0, %zmm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; GFNIAVX512-NEXT: retq %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a, <64 x i8> %a, <64 x i8> ) ret <64 x i8> %res diff --git a/llvm/test/CodeGen/X86/half.ll b/llvm/test/CodeGen/X86/half.ll index 596e465ee8cacf20365b1ffaee7aecf0570d6457..7225257203161b247a12437a574611374bb6efb4 100644 --- a/llvm/test/CodeGen/X86/half.ll +++ b/llvm/test/CodeGen/X86/half.ll @@ -1082,12 +1082,11 @@ define void @main.158() #0 { ; BWON-F16C-LABEL: main.158: ; BWON-F16C: # %bb.0: # %entry ; BWON-F16C-NEXT: vxorps %xmm0, %xmm0, %xmm0 -; BWON-F16C-NEXT: vcvtps2ph $4, %xmm0, %xmm0 -; BWON-F16C-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero -; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm0 -; BWON-F16C-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero -; BWON-F16C-NEXT: vucomiss %xmm0, %xmm1 -; BWON-F16C-NEXT: vxorps %xmm0, %xmm0, %xmm0 +; BWON-F16C-NEXT: vcvtps2ph $4, %xmm0, %xmm1 +; BWON-F16C-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero +; BWON-F16C-NEXT: vcvtph2ps %xmm1, %xmm1 +; BWON-F16C-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero +; BWON-F16C-NEXT: vucomiss %xmm1, %xmm2 ; BWON-F16C-NEXT: jae .LBB20_2 ; BWON-F16C-NEXT: # %bb.1: # %entry ; BWON-F16C-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero @@ -1100,8 +1099,7 @@ define void @main.158() #0 { ; CHECK-I686-LABEL: main.158: ; CHECK-I686: # %bb.0: # %entry ; CHECK-I686-NEXT: subl $12, %esp -; CHECK-I686-NEXT: pxor %xmm0, %xmm0 -; CHECK-I686-NEXT: movd %xmm0, (%esp) +; CHECK-I686-NEXT: movl $0, (%esp) ; CHECK-I686-NEXT: calll __truncsfhf2 ; CHECK-I686-NEXT: pextrw $0, %xmm0, %eax ; CHECK-I686-NEXT: movw %ax, (%esp) diff --git a/llvm/test/CodeGen/X86/ldexp.ll b/llvm/test/CodeGen/X86/ldexp.ll index 44c57c54ba0233a38d1c26e5e53d540c4ce4b550..ec128fc6686c85bb4ac518432791159fb720fc44 100644 --- a/llvm/test/CodeGen/X86/ldexp.ll +++ b/llvm/test/CodeGen/X86/ldexp.ll @@ -91,10 +91,11 @@ define double @ldexp_f64(i8 zeroext %x) { ; ; WIN32-LABEL: ldexp_f64: ; WIN32: # %bb.0: +; WIN32-NEXT: subl $12, %esp ; WIN32-NEXT: movzbl {{[0-9]+}}(%esp), %eax -; WIN32-NEXT: pushl %eax -; WIN32-NEXT: pushl $1072693248 # imm = 0x3FF00000 -; WIN32-NEXT: pushl $0 +; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp) +; WIN32-NEXT: fld1 +; WIN32-NEXT: fstpl (%esp) ; WIN32-NEXT: calll _ldexp ; WIN32-NEXT: addl $12, %esp ; WIN32-NEXT: retl diff --git a/llvm/test/CodeGen/X86/llvm.frexp.ll b/llvm/test/CodeGen/X86/llvm.frexp.ll index bd0a1dce6946c64e0ae85c1831d2b20019bc42f5..d6038cebd45822b9d858bfd9fc79dc4b18d8f595 100644 --- a/llvm/test/CodeGen/X86/llvm.frexp.ll +++ b/llvm/test/CodeGen/X86/llvm.frexp.ll @@ -2,24 +2,241 @@ ; RUN: llc -mtriple=x86_64-unknown-unknown < %s | FileCheck -check-prefixes=X64 %s ; RUN: llc -mtriple=i386-pc-win32 < %s | FileCheck -check-prefix=WIN32 %s -; FIXME -; define { half, i32 } @test_frexp_f16_i32(half %a) { -; %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) -; ret { half, i32 } %result -; } +define { half, i32 } @test_frexp_f16_i32(half %a) { +; X64-LABEL: test_frexp_f16_i32: +; X64: # %bb.0: +; X64-NEXT: subq $24, %rsp +; X64-NEXT: .cfi_def_cfa_offset 32 +; X64-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill +; X64-NEXT: callq __extendhfsf2@PLT +; X64-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: callq __truncsfhf2@PLT +; X64-NEXT: pextrw $0, %xmm0, %ecx +; X64-NEXT: movl %ecx, %eax +; X64-NEXT: andl $31744, %eax # imm = 0x7C00 +; X64-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload +; X64-NEXT: pextrw $0, %xmm0, %edx +; X64-NEXT: movl %edx, %esi +; X64-NEXT: andl $32767, %esi # imm = 0x7FFF +; X64-NEXT: cmpl $1024, %esi # imm = 0x400 +; X64-NEXT: cmovael %edx, %ecx +; X64-NEXT: cmovael %esi, %eax +; X64-NEXT: shrl $10, %eax +; X64-NEXT: leal -12(%rax), %edi +; X64-NEXT: cmpl $1024, %esi # imm = 0x400 +; X64-NEXT: cmovael %eax, %edi +; X64-NEXT: addl $-14, %edi +; X64-NEXT: andl $-31745, %ecx # imm = 0x83FF +; X64-NEXT: orl $14336, %ecx # imm = 0x3800 +; X64-NEXT: addl $-31744, %esi # imm = 0x8400 +; X64-NEXT: movzwl %si, %esi +; X64-NEXT: xorl %eax, %eax +; X64-NEXT: cmpl $33792, %esi # imm = 0x8400 +; X64-NEXT: cmoval %edi, %eax +; X64-NEXT: cmovbel %edx, %ecx +; X64-NEXT: pinsrw $0, %ecx, %xmm0 +; X64-NEXT: addq $24, %rsp +; X64-NEXT: .cfi_def_cfa_offset 8 +; X64-NEXT: retq +; +; WIN32-LABEL: test_frexp_f16_i32: +; WIN32: # %bb.0: +; WIN32-NEXT: pushl %esi +; WIN32-NEXT: subl $16, %esp +; WIN32-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; WIN32-NEXT: movl %eax, (%esp) +; WIN32-NEXT: calll ___gnu_h2f_ieee +; WIN32-NEXT: fld %st(0) +; WIN32-NEXT: fmuls __real@4c000000 +; WIN32-NEXT: fstps {{[0-9]+}}(%esp) +; WIN32-NEXT: fsts {{[0-9]+}}(%esp) +; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edx +; WIN32-NEXT: movl %edx, %ecx +; WIN32-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF +; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 +; WIN32-NEXT: jb LBB0_1 +; WIN32-NEXT: # %bb.2: +; WIN32-NEXT: movl %ecx, %eax +; WIN32-NEXT: jmp LBB0_3 +; WIN32-NEXT: LBB0_1: +; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edx +; WIN32-NEXT: movl %edx, %eax +; WIN32-NEXT: andl $2139095040, %eax # imm = 0x7F800000 +; WIN32-NEXT: LBB0_3: +; WIN32-NEXT: shrl $23, %eax +; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 +; WIN32-NEXT: jae LBB0_5 +; WIN32-NEXT: # %bb.4: +; WIN32-NEXT: addl $-25, %eax +; WIN32-NEXT: LBB0_5: +; WIN32-NEXT: andl $-2139095041, %edx # imm = 0x807FFFFF +; WIN32-NEXT: orl $1056964608, %edx # imm = 0x3F000000 +; WIN32-NEXT: movl %edx, {{[0-9]+}}(%esp) +; WIN32-NEXT: addl $-126, %eax +; WIN32-NEXT: addl $-2139095040, %ecx # imm = 0x80800000 +; WIN32-NEXT: flds {{[0-9]+}}(%esp) +; WIN32-NEXT: xorl %esi, %esi +; WIN32-NEXT: cmpl $-2139095040, %ecx # imm = 0x80800000 +; WIN32-NEXT: jbe LBB0_7 +; WIN32-NEXT: # %bb.6: +; WIN32-NEXT: fstp %st(1) +; WIN32-NEXT: fldz +; WIN32-NEXT: LBB0_7: +; WIN32-NEXT: fstp %st(0) +; WIN32-NEXT: fstps (%esp) +; WIN32-NEXT: jbe LBB0_9 +; WIN32-NEXT: # %bb.8: +; WIN32-NEXT: movl %eax, %esi +; WIN32-NEXT: LBB0_9: +; WIN32-NEXT: calll ___gnu_f2h_ieee +; WIN32-NEXT: movl %esi, %edx +; WIN32-NEXT: addl $16, %esp +; WIN32-NEXT: popl %esi +; WIN32-NEXT: retl + %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) + ret { half, i32 } %result +} -; define half @test_frexp_f16_i32_only_use_fract(half %a) { -; %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) -; %result.0 = extractvalue { half, i32 } %result, 0 -; ret half %result.0 -; } +define half @test_frexp_f16_i32_only_use_fract(half %a) { +; X64-LABEL: test_frexp_f16_i32_only_use_fract: +; X64: # %bb.0: +; X64-NEXT: subq $24, %rsp +; X64-NEXT: .cfi_def_cfa_offset 32 +; X64-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill +; X64-NEXT: callq __extendhfsf2@PLT +; X64-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: callq __truncsfhf2@PLT +; X64-NEXT: pextrw $0, %xmm0, %eax +; X64-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload +; X64-NEXT: pextrw $0, %xmm0, %ecx +; X64-NEXT: movl %ecx, %edx +; X64-NEXT: andl $32767, %edx # imm = 0x7FFF +; X64-NEXT: cmpl $1024, %edx # imm = 0x400 +; X64-NEXT: cmovael %ecx, %eax +; X64-NEXT: andl $-31745, %eax # imm = 0x83FF +; X64-NEXT: orl $14336, %eax # imm = 0x3800 +; X64-NEXT: addl $-31744, %edx # imm = 0x8400 +; X64-NEXT: movzwl %dx, %edx +; X64-NEXT: cmpl $33792, %edx # imm = 0x8400 +; X64-NEXT: cmovbel %ecx, %eax +; X64-NEXT: pinsrw $0, %eax, %xmm0 +; X64-NEXT: addq $24, %rsp +; X64-NEXT: .cfi_def_cfa_offset 8 +; X64-NEXT: retq +; +; WIN32-LABEL: test_frexp_f16_i32_only_use_fract: +; WIN32: # %bb.0: +; WIN32-NEXT: subl $16, %esp +; WIN32-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; WIN32-NEXT: movl %eax, (%esp) +; WIN32-NEXT: calll ___gnu_h2f_ieee +; WIN32-NEXT: fsts {{[0-9]+}}(%esp) +; WIN32-NEXT: fld %st(0) +; WIN32-NEXT: fmuls __real@4c000000 +; WIN32-NEXT: fstps {{[0-9]+}}(%esp) +; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ecx +; WIN32-NEXT: movl %ecx, %eax +; WIN32-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF +; WIN32-NEXT: cmpl $8388608, %eax # imm = 0x800000 +; WIN32-NEXT: jae LBB1_2 +; WIN32-NEXT: # %bb.1: +; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ecx +; WIN32-NEXT: LBB1_2: +; WIN32-NEXT: andl $-2139095041, %ecx # imm = 0x807FFFFF +; WIN32-NEXT: orl $1056964608, %ecx # imm = 0x3F000000 +; WIN32-NEXT: movl %ecx, {{[0-9]+}}(%esp) +; WIN32-NEXT: addl $-2139095040, %eax # imm = 0x80800000 +; WIN32-NEXT: flds {{[0-9]+}}(%esp) +; WIN32-NEXT: cmpl $-2139095040, %eax # imm = 0x80800000 +; WIN32-NEXT: jbe LBB1_4 +; WIN32-NEXT: # %bb.3: +; WIN32-NEXT: fstp %st(1) +; WIN32-NEXT: fldz +; WIN32-NEXT: LBB1_4: +; WIN32-NEXT: fstp %st(0) +; WIN32-NEXT: fstps (%esp) +; WIN32-NEXT: calll ___gnu_f2h_ieee +; WIN32-NEXT: addl $16, %esp +; WIN32-NEXT: retl + %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) + %result.0 = extractvalue { half, i32 } %result, 0 + ret half %result.0 +} -; define i32 @test_frexp_f16_i32_only_use_exp(half %a) { -; %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) -; %result.0 = extractvalue { half, i32 } %result, 1 -; ret i32 %result.0 -; } +define i32 @test_frexp_f16_i32_only_use_exp(half %a) { +; X64-LABEL: test_frexp_f16_i32_only_use_exp: +; X64: # %bb.0: +; X64-NEXT: subq $24, %rsp +; X64-NEXT: .cfi_def_cfa_offset 32 +; X64-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill +; X64-NEXT: callq __extendhfsf2@PLT +; X64-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: callq __truncsfhf2@PLT +; X64-NEXT: pextrw $0, %xmm0, %eax +; X64-NEXT: andl $31744, %eax # imm = 0x7C00 +; X64-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload +; X64-NEXT: pextrw $0, %xmm0, %ecx +; X64-NEXT: andl $32767, %ecx # imm = 0x7FFF +; X64-NEXT: cmpl $1024, %ecx # imm = 0x400 +; X64-NEXT: cmovael %ecx, %eax +; X64-NEXT: shrl $10, %eax +; X64-NEXT: leal -12(%rax), %edx +; X64-NEXT: cmpl $1024, %ecx # imm = 0x400 +; X64-NEXT: cmovael %eax, %edx +; X64-NEXT: addl $-14, %edx +; X64-NEXT: addl $-31744, %ecx # imm = 0x8400 +; X64-NEXT: movzwl %cx, %ecx +; X64-NEXT: xorl %eax, %eax +; X64-NEXT: cmpl $33792, %ecx # imm = 0x8400 +; X64-NEXT: cmoval %edx, %eax +; X64-NEXT: addq $24, %rsp +; X64-NEXT: .cfi_def_cfa_offset 8 +; X64-NEXT: retq +; +; WIN32-LABEL: test_frexp_f16_i32_only_use_exp: +; WIN32: # %bb.0: +; WIN32-NEXT: subl $8, %esp +; WIN32-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; WIN32-NEXT: pushl %eax +; WIN32-NEXT: calll ___gnu_h2f_ieee +; WIN32-NEXT: addl $4, %esp +; WIN32-NEXT: fld %st(0) +; WIN32-NEXT: fmuls __real@4c000000 +; WIN32-NEXT: fstps {{[0-9]+}}(%esp) +; WIN32-NEXT: fstps (%esp) +; WIN32-NEXT: movl $2147483647, %ecx # imm = 0x7FFFFFFF +; WIN32-NEXT: andl (%esp), %ecx +; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 +; WIN32-NEXT: jb LBB2_1 +; WIN32-NEXT: # %bb.2: +; WIN32-NEXT: movl %ecx, %edx +; WIN32-NEXT: jmp LBB2_3 +; WIN32-NEXT: LBB2_1: +; WIN32-NEXT: movl $2139095040, %edx # imm = 0x7F800000 +; WIN32-NEXT: andl {{[0-9]+}}(%esp), %edx +; WIN32-NEXT: LBB2_3: +; WIN32-NEXT: shrl $23, %edx +; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 +; WIN32-NEXT: jae LBB2_5 +; WIN32-NEXT: # %bb.4: +; WIN32-NEXT: addl $-25, %edx +; WIN32-NEXT: LBB2_5: +; WIN32-NEXT: addl $-2139095040, %ecx # imm = 0x80800000 +; WIN32-NEXT: xorl %eax, %eax +; WIN32-NEXT: cmpl $-2139095040, %ecx # imm = 0x80800000 +; WIN32-NEXT: jbe LBB2_7 +; WIN32-NEXT: # %bb.6: +; WIN32-NEXT: addl $-126, %edx +; WIN32-NEXT: movl %edx, %eax +; WIN32-NEXT: LBB2_7: +; WIN32-NEXT: addl $8, %esp +; WIN32-NEXT: retl + %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) + %result.0 = extractvalue { half, i32 } %result, 1 + ret i32 %result.0 +} +; FIXME ; define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) { ; %result = call { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half> %a) ; ret { <2 x half>, <2 x i32> } %result @@ -61,21 +278,21 @@ define { float, i32 } @test_frexp_f32_i32(float %a) { ; WIN32-NEXT: movl %eax, %ecx ; WIN32-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 -; WIN32-NEXT: jb LBB0_1 +; WIN32-NEXT: jb LBB3_1 ; WIN32-NEXT: # %bb.2: ; WIN32-NEXT: movl %ecx, %edx -; WIN32-NEXT: jmp LBB0_3 -; WIN32-NEXT: LBB0_1: +; WIN32-NEXT: jmp LBB3_3 +; WIN32-NEXT: LBB3_1: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax ; WIN32-NEXT: movl %eax, %edx ; WIN32-NEXT: andl $2139095040, %edx # imm = 0x7F800000 -; WIN32-NEXT: LBB0_3: +; WIN32-NEXT: LBB3_3: ; WIN32-NEXT: shrl $23, %edx ; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 -; WIN32-NEXT: jae LBB0_5 +; WIN32-NEXT: jae LBB3_5 ; WIN32-NEXT: # %bb.4: ; WIN32-NEXT: addl $-25, %edx -; WIN32-NEXT: LBB0_5: +; WIN32-NEXT: LBB3_5: ; WIN32-NEXT: andl $-2139095041, %eax # imm = 0x807FFFFF ; WIN32-NEXT: orl $1056964608, %eax # imm = 0x3F000000 ; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp) @@ -83,16 +300,16 @@ define { float, i32 } @test_frexp_f32_i32(float %a) { ; WIN32-NEXT: addl $-2139095040, %ecx # imm = 0x80800000 ; WIN32-NEXT: xorl %eax, %eax ; WIN32-NEXT: cmpl $-2139095040, %ecx # imm = 0x80800000 -; WIN32-NEXT: jbe LBB0_7 +; WIN32-NEXT: jbe LBB3_7 ; WIN32-NEXT: # %bb.6: ; WIN32-NEXT: movl %edx, %eax -; WIN32-NEXT: LBB0_7: +; WIN32-NEXT: LBB3_7: ; WIN32-NEXT: flds {{[0-9]+}}(%esp) -; WIN32-NEXT: jbe LBB0_9 +; WIN32-NEXT: jbe LBB3_9 ; WIN32-NEXT: # %bb.8: ; WIN32-NEXT: fstp %st(1) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB0_9: +; WIN32-NEXT: LBB3_9: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: addl $12, %esp ; WIN32-NEXT: retl @@ -123,21 +340,21 @@ define float @test_frexp_f32_i32_only_use_fract(float %a) { ; WIN32-NEXT: movl %ecx, %eax ; WIN32-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %eax # imm = 0x800000 -; WIN32-NEXT: jae LBB1_2 +; WIN32-NEXT: jae LBB4_2 ; WIN32-NEXT: # %bb.1: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; WIN32-NEXT: LBB1_2: +; WIN32-NEXT: LBB4_2: ; WIN32-NEXT: andl $-2139095041, %ecx # imm = 0x807FFFFF ; WIN32-NEXT: orl $1056964608, %ecx # imm = 0x3F000000 ; WIN32-NEXT: movl %ecx, {{[0-9]+}}(%esp) ; WIN32-NEXT: addl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: cmpl $-2139095040, %eax # imm = 0x80800000 -; WIN32-NEXT: jbe LBB1_4 +; WIN32-NEXT: jbe LBB4_4 ; WIN32-NEXT: # %bb.3: ; WIN32-NEXT: fstp %st(1) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB1_4: +; WIN32-NEXT: LBB4_4: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: addl $12, %esp ; WIN32-NEXT: retl @@ -169,28 +386,28 @@ define i32 @test_frexp_f32_i32_only_use_exp(float %a) { ; WIN32-NEXT: movl $2147483647, %ecx # imm = 0x7FFFFFFF ; WIN32-NEXT: andl (%esp), %ecx ; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 -; WIN32-NEXT: jb LBB2_1 +; WIN32-NEXT: jb LBB5_1 ; WIN32-NEXT: # %bb.2: ; WIN32-NEXT: movl %ecx, %edx -; WIN32-NEXT: jmp LBB2_3 -; WIN32-NEXT: LBB2_1: +; WIN32-NEXT: jmp LBB5_3 +; WIN32-NEXT: LBB5_1: ; WIN32-NEXT: movl $2139095040, %edx # imm = 0x7F800000 ; WIN32-NEXT: andl {{[0-9]+}}(%esp), %edx -; WIN32-NEXT: LBB2_3: +; WIN32-NEXT: LBB5_3: ; WIN32-NEXT: shrl $23, %edx ; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 -; WIN32-NEXT: jae LBB2_5 +; WIN32-NEXT: jae LBB5_5 ; WIN32-NEXT: # %bb.4: ; WIN32-NEXT: addl $-25, %edx -; WIN32-NEXT: LBB2_5: +; WIN32-NEXT: LBB5_5: ; WIN32-NEXT: addl $-2139095040, %ecx # imm = 0x80800000 ; WIN32-NEXT: xorl %eax, %eax ; WIN32-NEXT: cmpl $-2139095040, %ecx # imm = 0x80800000 -; WIN32-NEXT: jbe LBB2_7 +; WIN32-NEXT: jbe LBB5_7 ; WIN32-NEXT: # %bb.6: ; WIN32-NEXT: addl $-126, %edx ; WIN32-NEXT: movl %edx, %eax -; WIN32-NEXT: LBB2_7: +; WIN32-NEXT: LBB5_7: ; WIN32-NEXT: addl $8, %esp ; WIN32-NEXT: retl %result = call { float, i32 } @llvm.frexp.f32.i32(float %a) @@ -275,15 +492,15 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) { ; WIN32-NEXT: movl %ebx, %eax ; WIN32-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %eax # imm = 0x800000 -; WIN32-NEXT: jb LBB3_1 +; WIN32-NEXT: jb LBB6_1 ; WIN32-NEXT: # %bb.2: ; WIN32-NEXT: movl %eax, %ecx -; WIN32-NEXT: jmp LBB3_3 -; WIN32-NEXT: LBB3_1: +; WIN32-NEXT: jmp LBB6_3 +; WIN32-NEXT: LBB6_1: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ebx ; WIN32-NEXT: movl %ebx, %ecx ; WIN32-NEXT: andl $2139095040, %ecx # imm = 0x7F800000 -; WIN32-NEXT: LBB3_3: +; WIN32-NEXT: LBB6_3: ; WIN32-NEXT: shrl $23, %ecx ; WIN32-NEXT: cmpl $8388608, %eax # imm = 0x800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) @@ -291,24 +508,24 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) { ; WIN32-NEXT: fmul %st(3), %st ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) ; WIN32-NEXT: fsts {{[0-9]+}}(%esp) -; WIN32-NEXT: jae LBB3_5 +; WIN32-NEXT: jae LBB6_5 ; WIN32-NEXT: # %bb.4: ; WIN32-NEXT: addl $-25, %ecx -; WIN32-NEXT: LBB3_5: +; WIN32-NEXT: LBB6_5: ; WIN32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ecx ; WIN32-NEXT: movl %ecx, %edx ; WIN32-NEXT: andl $2147483647, %edx # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %edx # imm = 0x800000 -; WIN32-NEXT: jb LBB3_6 +; WIN32-NEXT: jb LBB6_6 ; WIN32-NEXT: # %bb.7: ; WIN32-NEXT: movl %edx, %esi -; WIN32-NEXT: jmp LBB3_8 -; WIN32-NEXT: LBB3_6: +; WIN32-NEXT: jmp LBB6_8 +; WIN32-NEXT: LBB6_6: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ecx ; WIN32-NEXT: movl %ecx, %esi ; WIN32-NEXT: andl $2139095040, %esi # imm = 0x7F800000 -; WIN32-NEXT: LBB3_8: +; WIN32-NEXT: LBB6_8: ; WIN32-NEXT: shrl $23, %esi ; WIN32-NEXT: cmpl $8388608, %edx # imm = 0x800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) @@ -316,24 +533,24 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) { ; WIN32-NEXT: fmul %st(4), %st ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) ; WIN32-NEXT: fsts {{[0-9]+}}(%esp) -; WIN32-NEXT: jae LBB3_10 +; WIN32-NEXT: jae LBB6_10 ; WIN32-NEXT: # %bb.9: ; WIN32-NEXT: addl $-25, %esi -; WIN32-NEXT: LBB3_10: +; WIN32-NEXT: LBB6_10: ; WIN32-NEXT: movl %esi, (%esp) # 4-byte Spill ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edi ; WIN32-NEXT: movl %edi, %ebp ; WIN32-NEXT: andl $2147483647, %ebp # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %ebp # imm = 0x800000 -; WIN32-NEXT: jb LBB3_11 +; WIN32-NEXT: jb LBB6_11 ; WIN32-NEXT: # %bb.12: ; WIN32-NEXT: movl %ebp, %esi -; WIN32-NEXT: jmp LBB3_13 -; WIN32-NEXT: LBB3_11: +; WIN32-NEXT: jmp LBB6_13 +; WIN32-NEXT: LBB6_11: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edi ; WIN32-NEXT: movl %edi, %esi ; WIN32-NEXT: andl $2139095040, %esi # imm = 0x7F800000 -; WIN32-NEXT: LBB3_13: +; WIN32-NEXT: LBB6_13: ; WIN32-NEXT: shrl $23, %esi ; WIN32-NEXT: cmpl $8388608, %ebp # imm = 0x800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) @@ -344,30 +561,30 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) { ; WIN32-NEXT: fsts {{[0-9]+}}(%esp) ; WIN32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; WIN32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; WIN32-NEXT: jae LBB3_15 +; WIN32-NEXT: jae LBB6_15 ; WIN32-NEXT: # %bb.14: ; WIN32-NEXT: addl $-25, %esi -; WIN32-NEXT: LBB3_15: +; WIN32-NEXT: LBB6_15: ; WIN32-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax ; WIN32-NEXT: movl %eax, %edx ; WIN32-NEXT: andl $2147483647, %edx # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %edx # imm = 0x800000 -; WIN32-NEXT: jb LBB3_16 +; WIN32-NEXT: jb LBB6_16 ; WIN32-NEXT: # %bb.17: ; WIN32-NEXT: movl %edx, %esi -; WIN32-NEXT: jmp LBB3_18 -; WIN32-NEXT: LBB3_16: +; WIN32-NEXT: jmp LBB6_18 +; WIN32-NEXT: LBB6_16: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax ; WIN32-NEXT: movl %eax, %esi ; WIN32-NEXT: andl $2139095040, %esi # imm = 0x7F800000 -; WIN32-NEXT: LBB3_18: +; WIN32-NEXT: LBB6_18: ; WIN32-NEXT: shrl $23, %esi ; WIN32-NEXT: cmpl $8388608, %edx # imm = 0x800000 -; WIN32-NEXT: jae LBB3_20 +; WIN32-NEXT: jae LBB6_20 ; WIN32-NEXT: # %bb.19: ; WIN32-NEXT: addl $-25, %esi -; WIN32-NEXT: LBB3_20: +; WIN32-NEXT: LBB6_20: ; WIN32-NEXT: andl $-2139095041, %eax # imm = 0x807FFFFF ; WIN32-NEXT: orl $1056964608, %eax # imm = 0x3F000000 ; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp) @@ -389,61 +606,61 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) { ; WIN32-NEXT: xorl %edi, %edi ; WIN32-NEXT: cmpl $-2139095040, %edx # imm = 0x80800000 ; WIN32-NEXT: movl $0, %ebx -; WIN32-NEXT: jbe LBB3_22 +; WIN32-NEXT: jbe LBB6_22 ; WIN32-NEXT: # %bb.21: ; WIN32-NEXT: movl %esi, %ebx -; WIN32-NEXT: LBB3_22: +; WIN32-NEXT: LBB6_22: ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; WIN32-NEXT: jbe LBB3_24 +; WIN32-NEXT: jbe LBB6_24 ; WIN32-NEXT: # %bb.23: ; WIN32-NEXT: fstp %st(1) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB3_24: +; WIN32-NEXT: LBB6_24: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: addl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: cmpl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: movl $0, %edx ; WIN32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload -; WIN32-NEXT: jbe LBB3_26 +; WIN32-NEXT: jbe LBB6_26 ; WIN32-NEXT: # %bb.25: ; WIN32-NEXT: movl %ecx, %edx -; WIN32-NEXT: LBB3_26: +; WIN32-NEXT: LBB6_26: ; WIN32-NEXT: flds {{[0-9]+}}(%esp) -; WIN32-NEXT: jbe LBB3_28 +; WIN32-NEXT: jbe LBB6_28 ; WIN32-NEXT: # %bb.27: ; WIN32-NEXT: fstp %st(3) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB3_28: +; WIN32-NEXT: LBB6_28: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; WIN32-NEXT: addl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: cmpl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: movl $0, %ecx -; WIN32-NEXT: jbe LBB3_30 +; WIN32-NEXT: jbe LBB6_30 ; WIN32-NEXT: # %bb.29: ; WIN32-NEXT: movl (%esp), %ecx # 4-byte Reload -; WIN32-NEXT: LBB3_30: +; WIN32-NEXT: LBB6_30: ; WIN32-NEXT: flds {{[0-9]+}}(%esp) -; WIN32-NEXT: jbe LBB3_32 +; WIN32-NEXT: jbe LBB6_32 ; WIN32-NEXT: # %bb.31: ; WIN32-NEXT: fstp %st(2) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB3_32: +; WIN32-NEXT: LBB6_32: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax ; WIN32-NEXT: addl $-2139095040, %ebp # imm = 0x80800000 ; WIN32-NEXT: cmpl $-2139095040, %ebp # imm = 0x80800000 -; WIN32-NEXT: jbe LBB3_34 +; WIN32-NEXT: jbe LBB6_34 ; WIN32-NEXT: # %bb.33: ; WIN32-NEXT: movl %esi, %edi -; WIN32-NEXT: LBB3_34: +; WIN32-NEXT: LBB6_34: ; WIN32-NEXT: flds {{[0-9]+}}(%esp) -; WIN32-NEXT: jbe LBB3_36 +; WIN32-NEXT: jbe LBB6_36 ; WIN32-NEXT: # %bb.35: ; WIN32-NEXT: fstp %st(4) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB3_36: +; WIN32-NEXT: LBB6_36: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: movl %edi, 28(%eax) ; WIN32-NEXT: movl %ecx, 24(%eax) @@ -531,10 +748,10 @@ define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) { ; WIN32-NEXT: movl %ecx, %eax ; WIN32-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %eax # imm = 0x800000 -; WIN32-NEXT: jae LBB4_2 +; WIN32-NEXT: jae LBB7_2 ; WIN32-NEXT: # %bb.1: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; WIN32-NEXT: LBB4_2: +; WIN32-NEXT: LBB7_2: ; WIN32-NEXT: andl $-2139095041, %ecx # imm = 0x807FFFFF ; WIN32-NEXT: orl $1056964608, %ecx # imm = 0x3F000000 ; WIN32-NEXT: movl %ecx, {{[0-9]+}}(%esp) @@ -542,10 +759,10 @@ define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) { ; WIN32-NEXT: movl %edx, %ecx ; WIN32-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 -; WIN32-NEXT: jae LBB4_4 +; WIN32-NEXT: jae LBB7_4 ; WIN32-NEXT: # %bb.3: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edx -; WIN32-NEXT: LBB4_4: +; WIN32-NEXT: LBB7_4: ; WIN32-NEXT: andl $-2139095041, %edx # imm = 0x807FFFFF ; WIN32-NEXT: orl $1056964608, %edx # imm = 0x3F000000 ; WIN32-NEXT: movl %edx, {{[0-9]+}}(%esp) @@ -553,10 +770,10 @@ define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) { ; WIN32-NEXT: movl %esi, %edx ; WIN32-NEXT: andl $2147483647, %edx # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %edx # imm = 0x800000 -; WIN32-NEXT: jae LBB4_6 +; WIN32-NEXT: jae LBB7_6 ; WIN32-NEXT: # %bb.5: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %esi -; WIN32-NEXT: LBB4_6: +; WIN32-NEXT: LBB7_6: ; WIN32-NEXT: andl $-2139095041, %esi # imm = 0x807FFFFF ; WIN32-NEXT: orl $1056964608, %esi # imm = 0x3F000000 ; WIN32-NEXT: movl %esi, {{[0-9]+}}(%esp) @@ -564,49 +781,49 @@ define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) { ; WIN32-NEXT: movl %edi, %esi ; WIN32-NEXT: andl $2147483647, %esi # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %esi # imm = 0x800000 -; WIN32-NEXT: jae LBB4_8 +; WIN32-NEXT: jae LBB7_8 ; WIN32-NEXT: # %bb.7: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edi -; WIN32-NEXT: LBB4_8: +; WIN32-NEXT: LBB7_8: ; WIN32-NEXT: andl $-2139095041, %edi # imm = 0x807FFFFF ; WIN32-NEXT: orl $1056964608, %edi # imm = 0x3F000000 ; WIN32-NEXT: movl %edi, {{[0-9]+}}(%esp) ; WIN32-NEXT: addl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: cmpl $-2139095040, %eax # imm = 0x80800000 -; WIN32-NEXT: jbe LBB4_10 +; WIN32-NEXT: jbe LBB7_10 ; WIN32-NEXT: # %bb.9: ; WIN32-NEXT: fstp %st(1) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB4_10: +; WIN32-NEXT: LBB7_10: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: addl $-2139095040, %ecx # imm = 0x80800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: cmpl $-2139095040, %ecx # imm = 0x80800000 -; WIN32-NEXT: jbe LBB4_12 +; WIN32-NEXT: jbe LBB7_12 ; WIN32-NEXT: # %bb.11: ; WIN32-NEXT: fstp %st(3) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB4_12: +; WIN32-NEXT: LBB7_12: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax ; WIN32-NEXT: addl $-2139095040, %edx # imm = 0x80800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: cmpl $-2139095040, %edx # imm = 0x80800000 -; WIN32-NEXT: jbe LBB4_14 +; WIN32-NEXT: jbe LBB7_14 ; WIN32-NEXT: # %bb.13: ; WIN32-NEXT: fstp %st(4) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB4_14: +; WIN32-NEXT: LBB7_14: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: addl $-2139095040, %esi # imm = 0x80800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: cmpl $-2139095040, %esi # imm = 0x80800000 -; WIN32-NEXT: jbe LBB4_16 +; WIN32-NEXT: jbe LBB7_16 ; WIN32-NEXT: # %bb.15: ; WIN32-NEXT: fstp %st(2) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB4_16: +; WIN32-NEXT: LBB7_16: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: fxch %st(1) ; WIN32-NEXT: fstps 12(%eax) @@ -673,14 +890,14 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) { ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax ; WIN32-NEXT: andl %edx, %eax ; WIN32-NEXT: cmpl $8388608, %eax # imm = 0x800000 -; WIN32-NEXT: jb LBB5_1 +; WIN32-NEXT: jb LBB8_1 ; WIN32-NEXT: # %bb.2: ; WIN32-NEXT: movl %eax, %esi -; WIN32-NEXT: jmp LBB5_3 -; WIN32-NEXT: LBB5_1: +; WIN32-NEXT: jmp LBB8_3 +; WIN32-NEXT: LBB8_1: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %esi ; WIN32-NEXT: andl %ecx, %esi -; WIN32-NEXT: LBB5_3: +; WIN32-NEXT: LBB8_3: ; WIN32-NEXT: shrl $23, %esi ; WIN32-NEXT: cmpl $8388608, %eax # imm = 0x800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) @@ -688,21 +905,21 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) { ; WIN32-NEXT: fmul %st(2), %st ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) -; WIN32-NEXT: jae LBB5_5 +; WIN32-NEXT: jae LBB8_5 ; WIN32-NEXT: # %bb.4: ; WIN32-NEXT: addl $-25, %esi -; WIN32-NEXT: LBB5_5: +; WIN32-NEXT: LBB8_5: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edi ; WIN32-NEXT: andl %edx, %edi ; WIN32-NEXT: cmpl $8388608, %edi # imm = 0x800000 -; WIN32-NEXT: jb LBB5_6 +; WIN32-NEXT: jb LBB8_6 ; WIN32-NEXT: # %bb.7: ; WIN32-NEXT: movl %edi, %ebx -; WIN32-NEXT: jmp LBB5_8 -; WIN32-NEXT: LBB5_6: +; WIN32-NEXT: jmp LBB8_8 +; WIN32-NEXT: LBB8_6: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ebx ; WIN32-NEXT: andl %ecx, %ebx -; WIN32-NEXT: LBB5_8: +; WIN32-NEXT: LBB8_8: ; WIN32-NEXT: shrl $23, %ebx ; WIN32-NEXT: cmpl $8388608, %edi # imm = 0x800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) @@ -710,22 +927,22 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) { ; WIN32-NEXT: fmul %st(2), %st ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) -; WIN32-NEXT: jae LBB5_10 +; WIN32-NEXT: jae LBB8_10 ; WIN32-NEXT: # %bb.9: ; WIN32-NEXT: addl $-25, %ebx -; WIN32-NEXT: LBB5_10: +; WIN32-NEXT: LBB8_10: ; WIN32-NEXT: movl %esi, (%esp) # 4-byte Spill ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ebp ; WIN32-NEXT: andl %edx, %ebp ; WIN32-NEXT: cmpl $8388608, %ebp # imm = 0x800000 -; WIN32-NEXT: jb LBB5_11 +; WIN32-NEXT: jb LBB8_11 ; WIN32-NEXT: # %bb.12: ; WIN32-NEXT: movl %ebp, %esi -; WIN32-NEXT: jmp LBB5_13 -; WIN32-NEXT: LBB5_11: +; WIN32-NEXT: jmp LBB8_13 +; WIN32-NEXT: LBB8_11: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %esi ; WIN32-NEXT: andl %ecx, %esi -; WIN32-NEXT: LBB5_13: +; WIN32-NEXT: LBB8_13: ; WIN32-NEXT: shrl $23, %esi ; WIN32-NEXT: cmpl $8388608, %ebp # imm = 0x800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) @@ -733,59 +950,59 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) { ; WIN32-NEXT: fxch %st(1) ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) -; WIN32-NEXT: jae LBB5_15 +; WIN32-NEXT: jae LBB8_15 ; WIN32-NEXT: # %bb.14: ; WIN32-NEXT: addl $-25, %esi -; WIN32-NEXT: LBB5_15: +; WIN32-NEXT: LBB8_15: ; WIN32-NEXT: andl {{[0-9]+}}(%esp), %edx ; WIN32-NEXT: cmpl $8388608, %edx # imm = 0x800000 -; WIN32-NEXT: jb LBB5_16 +; WIN32-NEXT: jb LBB8_16 ; WIN32-NEXT: # %bb.17: ; WIN32-NEXT: movl %edx, %ecx -; WIN32-NEXT: jmp LBB5_18 -; WIN32-NEXT: LBB5_16: +; WIN32-NEXT: jmp LBB8_18 +; WIN32-NEXT: LBB8_16: ; WIN32-NEXT: andl {{[0-9]+}}(%esp), %ecx -; WIN32-NEXT: LBB5_18: +; WIN32-NEXT: LBB8_18: ; WIN32-NEXT: shrl $23, %ecx ; WIN32-NEXT: cmpl $8388608, %edx # imm = 0x800000 -; WIN32-NEXT: jae LBB5_20 +; WIN32-NEXT: jae LBB8_20 ; WIN32-NEXT: # %bb.19: ; WIN32-NEXT: addl $-25, %ecx -; WIN32-NEXT: LBB5_20: +; WIN32-NEXT: LBB8_20: ; WIN32-NEXT: addl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: cmpl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: movl $0, %eax -; WIN32-NEXT: jbe LBB5_22 +; WIN32-NEXT: jbe LBB8_22 ; WIN32-NEXT: # %bb.21: ; WIN32-NEXT: movl (%esp), %eax # 4-byte Reload ; WIN32-NEXT: addl $-126, %eax -; WIN32-NEXT: LBB5_22: +; WIN32-NEXT: LBB8_22: ; WIN32-NEXT: movl %eax, (%esp) # 4-byte Spill ; WIN32-NEXT: addl $-2139095040, %edi # imm = 0x80800000 ; WIN32-NEXT: cmpl $-2139095040, %edi # imm = 0x80800000 ; WIN32-NEXT: movl $0, %edi -; WIN32-NEXT: jbe LBB5_24 +; WIN32-NEXT: jbe LBB8_24 ; WIN32-NEXT: # %bb.23: ; WIN32-NEXT: addl $-126, %ebx ; WIN32-NEXT: movl %ebx, %edi -; WIN32-NEXT: LBB5_24: +; WIN32-NEXT: LBB8_24: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax ; WIN32-NEXT: addl $-2139095040, %ebp # imm = 0x80800000 ; WIN32-NEXT: cmpl $-2139095040, %ebp # imm = 0x80800000 ; WIN32-NEXT: movl $0, %ebx -; WIN32-NEXT: jbe LBB5_26 +; WIN32-NEXT: jbe LBB8_26 ; WIN32-NEXT: # %bb.25: ; WIN32-NEXT: addl $-126, %esi ; WIN32-NEXT: movl %esi, %ebx -; WIN32-NEXT: LBB5_26: +; WIN32-NEXT: LBB8_26: ; WIN32-NEXT: addl $-2139095040, %edx # imm = 0x80800000 ; WIN32-NEXT: cmpl $-2139095040, %edx # imm = 0x80800000 ; WIN32-NEXT: movl $0, %edx -; WIN32-NEXT: jbe LBB5_28 +; WIN32-NEXT: jbe LBB8_28 ; WIN32-NEXT: # %bb.27: ; WIN32-NEXT: addl $-126, %ecx ; WIN32-NEXT: movl %ecx, %edx -; WIN32-NEXT: LBB5_28: +; WIN32-NEXT: LBB8_28: ; WIN32-NEXT: movl %edx, 12(%eax) ; WIN32-NEXT: movl %ebx, 8(%eax) ; WIN32-NEXT: movl %edi, 4(%eax) diff --git a/llvm/test/CodeGen/X86/memset64-on-x86-32.ll b/llvm/test/CodeGen/X86/memset64-on-x86-32.ll index c6eecdcdf99cc52a6ffd030981d7b44fce0f896a..480a0970bd39ddd5ba21db1218b6c5af7b1fa8bf 100644 --- a/llvm/test/CodeGen/X86/memset64-on-x86-32.ll +++ b/llvm/test/CodeGen/X86/memset64-on-x86-32.ll @@ -18,26 +18,17 @@ define void @bork(ptr nocapture align 4 %dst) nounwind { ; SLOW_32-LABEL: bork: ; SLOW_32: # %bb.0: ; SLOW_32-NEXT: movl {{[0-9]+}}(%esp), %eax -; SLOW_32-NEXT: movl $0, 4(%eax) -; SLOW_32-NEXT: movl $0, (%eax) -; SLOW_32-NEXT: movl $0, 12(%eax) -; SLOW_32-NEXT: movl $0, 8(%eax) -; SLOW_32-NEXT: movl $0, 20(%eax) -; SLOW_32-NEXT: movl $0, 16(%eax) -; SLOW_32-NEXT: movl $0, 28(%eax) -; SLOW_32-NEXT: movl $0, 24(%eax) -; SLOW_32-NEXT: movl $0, 36(%eax) -; SLOW_32-NEXT: movl $0, 32(%eax) -; SLOW_32-NEXT: movl $0, 44(%eax) -; SLOW_32-NEXT: movl $0, 40(%eax) -; SLOW_32-NEXT: movl $0, 52(%eax) -; SLOW_32-NEXT: movl $0, 48(%eax) -; SLOW_32-NEXT: movl $0, 60(%eax) -; SLOW_32-NEXT: movl $0, 56(%eax) -; SLOW_32-NEXT: movl $0, 68(%eax) -; SLOW_32-NEXT: movl $0, 64(%eax) -; SLOW_32-NEXT: movl $0, 76(%eax) -; SLOW_32-NEXT: movl $0, 72(%eax) +; SLOW_32-NEXT: xorps %xmm0, %xmm0 +; SLOW_32-NEXT: movsd %xmm0, 72(%eax) +; SLOW_32-NEXT: movsd %xmm0, 64(%eax) +; SLOW_32-NEXT: movsd %xmm0, 56(%eax) +; SLOW_32-NEXT: movsd %xmm0, 48(%eax) +; SLOW_32-NEXT: movsd %xmm0, 40(%eax) +; SLOW_32-NEXT: movsd %xmm0, 32(%eax) +; SLOW_32-NEXT: movsd %xmm0, 24(%eax) +; SLOW_32-NEXT: movsd %xmm0, 16(%eax) +; SLOW_32-NEXT: movsd %xmm0, 8(%eax) +; SLOW_32-NEXT: movsd %xmm0, (%eax) ; SLOW_32-NEXT: retl ; ; SLOW_64-LABEL: bork: diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll index 7dd4af7689792959a8c7986445ba466d69d16cc3..5cd0c232de44d498612ebcbf74103a786127b5aa 100644 --- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll +++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll @@ -2010,7 +2010,7 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-v ; CHECK: # %bb.0: ; CHECK-NEXT: vpsllw $4, %ymm0, %ymm1 ; CHECK-NEXT: vpsrlw $4, %ymm0, %ymm0 -; CHECK-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; CHECK-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; CHECK-NEXT: retq %shl = shl <32 x i8> %a, %lshr = lshr <32 x i8> %a, @@ -2023,7 +2023,7 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-le ; CHECK: # %bb.0: ; CHECK-NEXT: vpsllw $4, %ymm0, %ymm1 ; CHECK-NEXT: vpsrlw $4, %ymm0, %ymm0 -; CHECK-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; CHECK-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; CHECK-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0 ; CHECK-NEXT: retq %shl = shl <32 x i8> %a, diff --git a/llvm/test/CodeGen/X86/nocx16.ll b/llvm/test/CodeGen/X86/nocx16.ll deleted file mode 100644 index ec8e6b2c8c6ac0b5a0fdb77613afca652951d44f..0000000000000000000000000000000000000000 --- a/llvm/test/CodeGen/X86/nocx16.ll +++ /dev/null @@ -1,21 +0,0 @@ -; RUN: llc < %s -mtriple=x86_64-- -mcpu=corei7 -mattr=-cx16 | FileCheck %s -define void @test(ptr %a) nounwind { -entry: -; CHECK: __sync_val_compare_and_swap_16 - %0 = cmpxchg ptr %a, i128 1, i128 1 seq_cst seq_cst -; CHECK: __sync_lock_test_and_set_16 - %1 = atomicrmw xchg ptr %a, i128 1 seq_cst -; CHECK: __sync_fetch_and_add_16 - %2 = atomicrmw add ptr %a, i128 1 seq_cst -; CHECK: __sync_fetch_and_sub_16 - %3 = atomicrmw sub ptr %a, i128 1 seq_cst -; CHECK: __sync_fetch_and_and_16 - %4 = atomicrmw and ptr %a, i128 1 seq_cst -; CHECK: __sync_fetch_and_nand_16 - %5 = atomicrmw nand ptr %a, i128 1 seq_cst -; CHECK: __sync_fetch_and_or_16 - %6 = atomicrmw or ptr %a, i128 1 seq_cst -; CHECK: __sync_fetch_and_xor_16 - %7 = atomicrmw xor ptr %a, i128 1 seq_cst - ret void -} diff --git a/llvm/test/CodeGen/X86/nontemporal-3.ll b/llvm/test/CodeGen/X86/nontemporal-3.ll index a2d2c5ca43011863d576e1c12b35301cf0140f8d..f9872b10097a1501652b87f12b18836817fdd7b0 100644 --- a/llvm/test/CodeGen/X86/nontemporal-3.ll +++ b/llvm/test/CodeGen/X86/nontemporal-3.ll @@ -93,247 +93,66 @@ define void @test_zero_v4f64_align1(ptr %dst) nounwind { } define void @test_zero_v8f32_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v8f32_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v8f32_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorl %eax, %eax -; SSE4A-NEXT: movntiq %rax, 8(%rdi) -; SSE4A-NEXT: movntiq %rax, 24(%rdi) -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v8f32_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v8f32_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v8f32_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v8f32_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: retq store <8 x float> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v4i64_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v4i64_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v4i64_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v4i64_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v4i64_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v4i64_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v4i64_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: retq store <4 x i64> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v8i32_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v8i32_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v8i32_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v8i32_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v8i32_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v8i32_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v8i32_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: retq store <8 x i32> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v16i16_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v16i16_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v16i16_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v16i16_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v16i16_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v16i16_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v16i16_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: retq store <16 x i16> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v32i8_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v32i8_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v32i8_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v32i8_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v32i8_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v32i8_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v32i8_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: retq store <32 x i8> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } @@ -508,347 +327,86 @@ define void @test_zero_v8f64_align1(ptr %dst) nounwind { } define void @test_zero_v16f32_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v16f32_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: movntiq %rax, 40(%rdi) -; SSE2-NEXT: movntiq %rax, 32(%rdi) -; SSE2-NEXT: movntiq %rax, 56(%rdi) -; SSE2-NEXT: movntiq %rax, 48(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v16f32_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorl %eax, %eax -; SSE4A-NEXT: movntiq %rax, 8(%rdi) -; SSE4A-NEXT: movntiq %rax, 24(%rdi) -; SSE4A-NEXT: movntiq %rax, 40(%rdi) -; SSE4A-NEXT: movntiq %rax, 56(%rdi) -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 32(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 48(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v16f32_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: movntiq %rax, 40(%rdi) -; SSE41-NEXT: movntiq %rax, 32(%rdi) -; SSE41-NEXT: movntiq %rax, 56(%rdi) -; SSE41-NEXT: movntiq %rax, 48(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v16f32_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: movntiq %rax, 40(%rdi) -; AVX-NEXT: movntiq %rax, 32(%rdi) -; AVX-NEXT: movntiq %rax, 56(%rdi) -; AVX-NEXT: movntiq %rax, 48(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v16f32_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: movntiq %rax, 40(%rdi) -; AVX512-NEXT: movntiq %rax, 32(%rdi) -; AVX512-NEXT: movntiq %rax, 56(%rdi) -; AVX512-NEXT: movntiq %rax, 48(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v16f32_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: movntiq %rax, 40(%rdi) +; CHECK-NEXT: movntiq %rax, 32(%rdi) +; CHECK-NEXT: movntiq %rax, 56(%rdi) +; CHECK-NEXT: movntiq %rax, 48(%rdi) +; CHECK-NEXT: retq store <16 x float> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v8i64_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v8i64_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: movntiq %rax, 40(%rdi) -; SSE2-NEXT: movntiq %rax, 32(%rdi) -; SSE2-NEXT: movntiq %rax, 56(%rdi) -; SSE2-NEXT: movntiq %rax, 48(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v8i64_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 40(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 32(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 56(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 48(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v8i64_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: movntiq %rax, 40(%rdi) -; SSE41-NEXT: movntiq %rax, 32(%rdi) -; SSE41-NEXT: movntiq %rax, 56(%rdi) -; SSE41-NEXT: movntiq %rax, 48(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v8i64_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: movntiq %rax, 40(%rdi) -; AVX-NEXT: movntiq %rax, 32(%rdi) -; AVX-NEXT: movntiq %rax, 56(%rdi) -; AVX-NEXT: movntiq %rax, 48(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v8i64_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: movntiq %rax, 40(%rdi) -; AVX512-NEXT: movntiq %rax, 32(%rdi) -; AVX512-NEXT: movntiq %rax, 56(%rdi) -; AVX512-NEXT: movntiq %rax, 48(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v8i64_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: movntiq %rax, 40(%rdi) +; CHECK-NEXT: movntiq %rax, 32(%rdi) +; CHECK-NEXT: movntiq %rax, 56(%rdi) +; CHECK-NEXT: movntiq %rax, 48(%rdi) +; CHECK-NEXT: retq store <8 x i64> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v16i32_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v16i32_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: movntiq %rax, 40(%rdi) -; SSE2-NEXT: movntiq %rax, 32(%rdi) -; SSE2-NEXT: movntiq %rax, 56(%rdi) -; SSE2-NEXT: movntiq %rax, 48(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v16i32_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 40(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 32(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 56(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 48(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v16i32_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: movntiq %rax, 40(%rdi) -; SSE41-NEXT: movntiq %rax, 32(%rdi) -; SSE41-NEXT: movntiq %rax, 56(%rdi) -; SSE41-NEXT: movntiq %rax, 48(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v16i32_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: movntiq %rax, 40(%rdi) -; AVX-NEXT: movntiq %rax, 32(%rdi) -; AVX-NEXT: movntiq %rax, 56(%rdi) -; AVX-NEXT: movntiq %rax, 48(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v16i32_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: movntiq %rax, 40(%rdi) -; AVX512-NEXT: movntiq %rax, 32(%rdi) -; AVX512-NEXT: movntiq %rax, 56(%rdi) -; AVX512-NEXT: movntiq %rax, 48(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v16i32_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: movntiq %rax, 40(%rdi) +; CHECK-NEXT: movntiq %rax, 32(%rdi) +; CHECK-NEXT: movntiq %rax, 56(%rdi) +; CHECK-NEXT: movntiq %rax, 48(%rdi) +; CHECK-NEXT: retq store <16 x i32> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v32i16_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v32i16_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: movntiq %rax, 40(%rdi) -; SSE2-NEXT: movntiq %rax, 32(%rdi) -; SSE2-NEXT: movntiq %rax, 56(%rdi) -; SSE2-NEXT: movntiq %rax, 48(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v32i16_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 40(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 32(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 56(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 48(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v32i16_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: movntiq %rax, 40(%rdi) -; SSE41-NEXT: movntiq %rax, 32(%rdi) -; SSE41-NEXT: movntiq %rax, 56(%rdi) -; SSE41-NEXT: movntiq %rax, 48(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v32i16_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: movntiq %rax, 40(%rdi) -; AVX-NEXT: movntiq %rax, 32(%rdi) -; AVX-NEXT: movntiq %rax, 56(%rdi) -; AVX-NEXT: movntiq %rax, 48(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v32i16_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: movntiq %rax, 40(%rdi) -; AVX512-NEXT: movntiq %rax, 32(%rdi) -; AVX512-NEXT: movntiq %rax, 56(%rdi) -; AVX512-NEXT: movntiq %rax, 48(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v32i16_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: movntiq %rax, 40(%rdi) +; CHECK-NEXT: movntiq %rax, 32(%rdi) +; CHECK-NEXT: movntiq %rax, 56(%rdi) +; CHECK-NEXT: movntiq %rax, 48(%rdi) +; CHECK-NEXT: retq store <32 x i16> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v64i8_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v64i8_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: movntiq %rax, 40(%rdi) -; SSE2-NEXT: movntiq %rax, 32(%rdi) -; SSE2-NEXT: movntiq %rax, 56(%rdi) -; SSE2-NEXT: movntiq %rax, 48(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v64i8_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 40(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 32(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 56(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 48(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v64i8_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: movntiq %rax, 40(%rdi) -; SSE41-NEXT: movntiq %rax, 32(%rdi) -; SSE41-NEXT: movntiq %rax, 56(%rdi) -; SSE41-NEXT: movntiq %rax, 48(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v64i8_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: movntiq %rax, 40(%rdi) -; AVX-NEXT: movntiq %rax, 32(%rdi) -; AVX-NEXT: movntiq %rax, 56(%rdi) -; AVX-NEXT: movntiq %rax, 48(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v64i8_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: movntiq %rax, 40(%rdi) -; AVX512-NEXT: movntiq %rax, 32(%rdi) -; AVX512-NEXT: movntiq %rax, 56(%rdi) -; AVX512-NEXT: movntiq %rax, 48(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v64i8_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: movntiq %rax, 40(%rdi) +; CHECK-NEXT: movntiq %rax, 32(%rdi) +; CHECK-NEXT: movntiq %rax, 56(%rdi) +; CHECK-NEXT: movntiq %rax, 48(%rdi) +; CHECK-NEXT: retq store <64 x i8> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } @@ -1214,3 +772,7 @@ define void @test_zero_v64i8_align32(ptr %dst) nounwind { } !1 = !{i32 1} +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; SSE2: {{.*}} +; SSE41: {{.*}} +; SSE4A: {{.*}} diff --git a/llvm/test/CodeGen/X86/pr38738.ll b/llvm/test/CodeGen/X86/pr38738.ll index 753b7ce33d2bebac49b735539c33abf2cfcb40e3..205849e7d05db749c56d4bd77cfb60d3757e20f6 100644 --- a/llvm/test/CodeGen/X86/pr38738.ll +++ b/llvm/test/CodeGen/X86/pr38738.ll @@ -130,22 +130,15 @@ define void @tryset(ptr nocapture %x) { ; X86SSE2-LABEL: tryset: ; X86SSE2: # %bb.0: ; X86SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86SSE2-NEXT: movl $0, 4(%eax) -; X86SSE2-NEXT: movl $0, (%eax) -; X86SSE2-NEXT: movl $0, 12(%eax) -; X86SSE2-NEXT: movl $0, 8(%eax) -; X86SSE2-NEXT: movl $0, 20(%eax) -; X86SSE2-NEXT: movl $0, 16(%eax) -; X86SSE2-NEXT: movl $0, 28(%eax) -; X86SSE2-NEXT: movl $0, 24(%eax) -; X86SSE2-NEXT: movl $0, 36(%eax) -; X86SSE2-NEXT: movl $0, 32(%eax) -; X86SSE2-NEXT: movl $0, 44(%eax) -; X86SSE2-NEXT: movl $0, 40(%eax) -; X86SSE2-NEXT: movl $0, 52(%eax) -; X86SSE2-NEXT: movl $0, 48(%eax) -; X86SSE2-NEXT: movl $0, 60(%eax) -; X86SSE2-NEXT: movl $0, 56(%eax) +; X86SSE2-NEXT: xorps %xmm0, %xmm0 +; X86SSE2-NEXT: movsd %xmm0, 56(%eax) +; X86SSE2-NEXT: movsd %xmm0, 48(%eax) +; X86SSE2-NEXT: movsd %xmm0, 40(%eax) +; X86SSE2-NEXT: movsd %xmm0, 32(%eax) +; X86SSE2-NEXT: movsd %xmm0, 24(%eax) +; X86SSE2-NEXT: movsd %xmm0, 16(%eax) +; X86SSE2-NEXT: movsd %xmm0, 8(%eax) +; X86SSE2-NEXT: movsd %xmm0, (%eax) ; X86SSE2-NEXT: retl ; ; X64AVX-LABEL: tryset: diff --git a/llvm/test/CodeGen/X86/pr41619.ll b/llvm/test/CodeGen/X86/pr41619.ll index 7d1d139a38a520dce180b0f207c6e0fc10221f7d..88dcd7798f0c3d2cd7ad3dd068553c4ebb46a75a 100644 --- a/llvm/test/CodeGen/X86/pr41619.ll +++ b/llvm/test/CodeGen/X86/pr41619.ll @@ -7,10 +7,9 @@ define void @foo(double %arg) { ; CHECK: ## %bb.0: ## %bb ; CHECK-NEXT: vmovq %xmm0, %rax ; CHECK-NEXT: vmovd %eax, %xmm0 -; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1 ; CHECK-NEXT: vmovq %xmm0, %rax ; CHECK-NEXT: movl %eax, (%rax) -; CHECK-NEXT: vmovlps %xmm1, (%rax) +; CHECK-NEXT: movq $0, (%rax) ; CHECK-NEXT: retq bb: %tmp = bitcast double %arg to i64 diff --git a/llvm/test/CodeGen/X86/pr74736.ll b/llvm/test/CodeGen/X86/pr74736.ll new file mode 100644 index 0000000000000000000000000000000000000000..3dfdbf102c953e3b2f557c8d7d8a21f584edebc4 --- /dev/null +++ b/llvm/test/CodeGen/X86/pr74736.ll @@ -0,0 +1,68 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX + +define void @main(<16 x i32> %0, i32 %1) { +; SSE-LABEL: main: +; SSE: # %bb.0: # %entry +; SSE-NEXT: movd %edi, %xmm4 +; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,0,0,0] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm4[1,0] +; SSE-NEXT: paddd %xmm0, %xmm0 +; SSE-NEXT: paddd %xmm1, %xmm1 +; SSE-NEXT: paddd %xmm3, %xmm3 +; SSE-NEXT: paddd %xmm2, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,1,1,3] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,0],xmm1[1,0] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[1,3] +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,0],xmm2[1,0] +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[1,3] +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,0],xmm3[1,0] +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[1,3] +; SSE-NEXT: xorps %xmm2, %xmm0 +; SSE-NEXT: xorps %xmm4, %xmm1 +; SSE-NEXT: xorps %xmm0, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3] +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] +; SSE-NEXT: pxor %xmm0, %xmm1 +; SSE-NEXT: movd %xmm1, 0 +; SSE-NEXT: retq +; +; AVX-LABEL: main: +; AVX: # %bb.0: # %entry +; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0],xmm0[1,2,3] +; AVX-NEXT: movl $1, %eax +; AVX-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2 +; AVX-NEXT: vpinsrd $3, %edi, %xmm2, %xmm2 +; AVX-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vpaddd %ymm0, %ymm0, %ymm0 +; AVX-NEXT: vpaddd %ymm1, %ymm1, %ymm1 +; AVX-NEXT: vmovdqa {{.*#+}} ymm2 = [0,1,1,3,3,5,5,7] +; AVX-NEXT: vpermd %ymm0, %ymm2, %ymm2 +; AVX-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] +; AVX-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[3,3,3,3,7,7,7,7] +; AVX-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,1,3,4,5,5,7] +; AVX-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7] +; AVX-NEXT: vpxor %ymm0, %ymm2, %ymm0 +; AVX-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmovd %xmm0, 0 +; AVX-NEXT: vzeroupper +; AVX-NEXT: retq +entry: + %2 = insertelement <16 x i32> %0, i32 1, i64 1 + %3 = insertelement <16 x i32> %2, i32 %1, i64 3 + %4 = insertelement <16 x i32> %3, i32 0, i64 0 + %5 = shl <16 x i32> %4, + %6 = shufflevector <16 x i32> %5, <16 x i32> zeroinitializer, <16 x i32> + %7 = tail call i32 @llvm.vector.reduce.xor.v16i32(<16 x i32> %6) + store i32 %7, ptr null, align 4 + ret void +} +declare i32 @llvm.vector.reduce.xor.v16i32(<16 x i32>) diff --git a/llvm/test/CodeGen/X86/slow-unaligned-mem.ll b/llvm/test/CodeGen/X86/slow-unaligned-mem.ll index 85afa83e3f08e7a93e5b8d406144c95c13f66976..d74d195439bdae4c03ced89692d6c7dc976f9c0c 100644 --- a/llvm/test/CodeGen/X86/slow-unaligned-mem.ll +++ b/llvm/test/CodeGen/X86/slow-unaligned-mem.ll @@ -1,16 +1,16 @@ ; Intel chips with slow unaligned memory accesses -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium3 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium3m 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium-m 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium4 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium4m 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=yonah 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=prescott 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=nocona 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=core2 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=penryn 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=bonnell 2>&1 | FileCheck %s --check-prefixes=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium3 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SCALAR +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium3m 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SCALAR +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium-m 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium4 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium4m 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=yonah 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=prescott 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=nocona 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=core2 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=penryn 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=bonnell 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE ; Intel chips with fast unaligned memory accesses @@ -26,15 +26,15 @@ ; AMD chips with slow unaligned memory accesses -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-4 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-xp 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=k8 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=opteron 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon64 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-fx 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=k8-sse3 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=opteron-sse3 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon64-sse3 2>&1 | FileCheck %s --check-prefixes=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-4 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SCALAR +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-xp 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SCALAR +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=k8 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=opteron 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon64 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-fx 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=k8-sse3 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=opteron-sse3 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon64-sse3 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE ; AMD chips with fast unaligned memory accesses @@ -67,26 +67,40 @@ ; SLOW-NOT: not a recognized processor ; FAST-NOT: not a recognized processor define void @store_zeros(ptr %a) { -; SLOW-LABEL: store_zeros: -; SLOW: # %bb.0: -; SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NOT: movl +; SLOW-SCALAR-LABEL: store_zeros: +; SLOW-SCALAR: # %bb.0: +; SLOW-SCALAR-NEXT: movl {{[0-9]+}}(%esp), %eax +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NOT: movl +; +; SLOW-SSE-LABEL: store_zeros: +; SLOW-SSE: # %bb.0: +; SLOW-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; SLOW-SSE-NEXT: xorps %xmm0, %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NOT: movsd ; ; FAST-SSE-LABEL: store_zeros: ; FAST-SSE: # %bb.0: diff --git a/llvm/test/CodeGen/X86/var-permute-256.ll b/llvm/test/CodeGen/X86/var-permute-256.ll index 6c07c4ca523f8f6bf1b1658d38f4318258bc9083..853d8a278960d7e65d168edf2412c4972fff26f8 100644 --- a/llvm/test/CodeGen/X86/var-permute-256.ll +++ b/llvm/test/CodeGen/X86/var-permute-256.ll @@ -25,18 +25,20 @@ define <4 x i64> @var_shuffle_v4i64(<4 x i64> %v, <4 x i64> %indices) nounwind { ; ; AVX1-LABEL: var_shuffle_v4i64: ; AVX1: # %bb.0: -; AVX1-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3] -; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm3 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vpaddq %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vmovddup {{.*#+}} xmm3 = [2,2] +; AVX1-NEXT: # xmm3 = mem[0,0] +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm4 ; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm4 -; AVX1-NEXT: vpermilpd %ymm4, %ymm2, %ymm2 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3 +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm0[2,3,2,3] +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpermilpd %ymm1, %ymm4, %ymm2 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX1-NEXT: vpermilpd %ymm4, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1 -; AVX1-NEXT: vblendvpd %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vpermilpd %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vblendvpd %ymm3, %ymm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: var_shuffle_v4i64: @@ -88,15 +90,16 @@ define <8 x i32> @var_shuffle_v8i32(<8 x i32> %v, <8 x i32> %indices) nounwind { ; ; AVX1-LABEL: var_shuffle_v8i32: ; AVX1: # %bb.0: -; AVX1-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3] -; AVX1-NEXT: vpermilps %ymm1, %ymm2, %ymm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [3,3,3,3] +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3] +; AVX1-NEXT: vpermilps %ymm1, %ymm3, %ymm3 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 ; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1 -; AVX1-NEXT: vblendvps %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvps %ymm2, %ymm3, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; INT256-LABEL: var_shuffle_v8i32: @@ -445,18 +448,20 @@ define <4 x double> @var_shuffle_v4f64(<4 x double> %v, <4 x i64> %indices) noun ; ; AVX1-LABEL: var_shuffle_v4f64: ; AVX1: # %bb.0: -; AVX1-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3] -; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm3 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vpaddq %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vmovddup {{.*#+}} xmm3 = [2,2] +; AVX1-NEXT: # xmm3 = mem[0,0] +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm4 ; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm4 -; AVX1-NEXT: vpermilpd %ymm4, %ymm2, %ymm2 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3 +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm0[2,3,2,3] +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpermilpd %ymm1, %ymm4, %ymm2 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX1-NEXT: vpermilpd %ymm4, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1 -; AVX1-NEXT: vblendvpd %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vpermilpd %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vblendvpd %ymm3, %ymm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: var_shuffle_v4f64: @@ -508,15 +513,16 @@ define <8 x float> @var_shuffle_v8f32(<8 x float> %v, <8 x i32> %indices) nounwi ; ; AVX1-LABEL: var_shuffle_v8f32: ; AVX1: # %bb.0: -; AVX1-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3] -; AVX1-NEXT: vpermilps %ymm1, %ymm2, %ymm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [3,3,3,3] +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3] +; AVX1-NEXT: vpermilps %ymm1, %ymm3, %ymm3 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 ; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1 -; AVX1-NEXT: vblendvps %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvps %ymm2, %ymm3, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; INT256-LABEL: var_shuffle_v8f32: @@ -569,17 +575,19 @@ define <4 x i64> @var_shuffle_v4i64_from_v2i64(<2 x i64> %v, <4 x i64> %indices) ; AVX1-LABEL: var_shuffle_v4i64_from_v2i64: ; AVX1: # %bb.0: ; AVX1-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0 -; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm2 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vpaddq %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vmovddup {{.*#+}} xmm3 = [2,2] +; AVX1-NEXT: # xmm3 = mem[0,0] +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm4 ; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm3 -; AVX1-NEXT: vpermilpd %ymm3, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 -; AVX1-NEXT: vpermilpd %ymm3, %ymm0, %ymm2 -; AVX1-NEXT: vblendvpd %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpermilpd %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vpermilpd %ymm1, %ymm0, %ymm1 +; AVX1-NEXT: vblendvpd %ymm3, %ymm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: var_shuffle_v4i64_from_v2i64: @@ -633,14 +641,15 @@ define <8 x i32> @var_shuffle_v8i32_from_v4i32(<4 x i32> %v, <8 x i32> %indices) ; AVX1-LABEL: var_shuffle_v8i32_from_v4i32: ; AVX1: # %bb.0: # %entry ; AVX1-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0 -; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [3,3,3,3] +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm3 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 ; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1 -; AVX1-NEXT: vblendvps %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvps %ymm2, %ymm3, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; INT256-LABEL: var_shuffle_v8i32_from_v4i32: @@ -990,17 +999,19 @@ define <4 x double> @var_shuffle_v4f64_from_v2f64(<2 x double> %v, <4 x i64> %in ; AVX1-LABEL: var_shuffle_v4f64_from_v2f64: ; AVX1: # %bb.0: ; AVX1-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0 -; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm2 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vpaddq %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vmovddup {{.*#+}} xmm3 = [2,2] +; AVX1-NEXT: # xmm3 = mem[0,0] +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm4 ; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm3 -; AVX1-NEXT: vpermilpd %ymm3, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 -; AVX1-NEXT: vpermilpd %ymm3, %ymm0, %ymm2 -; AVX1-NEXT: vblendvpd %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpermilpd %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vpermilpd %ymm1, %ymm0, %ymm1 +; AVX1-NEXT: vblendvpd %ymm3, %ymm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: var_shuffle_v4f64_from_v2f64: @@ -1054,14 +1065,15 @@ define <8 x float> @var_shuffle_v8f32_from_v4f32(<4 x float> %v, <8 x i32> %indi ; AVX1-LABEL: var_shuffle_v8f32_from_v4f32: ; AVX1: # %bb.0: # %entry ; AVX1-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0 -; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [3,3,3,3] +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm3 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 ; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1 -; AVX1-NEXT: vblendvps %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvps %ymm2, %ymm3, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; INT256-LABEL: var_shuffle_v8f32_from_v4f32: @@ -1271,20 +1283,22 @@ define <4 x i64> @var_shuffle_v4i64_with_v16i8_indices(<4 x i64> %v, <16 x i8> % ; ; AVX1-LABEL: var_shuffle_v4i64_with_v16i8_indices: ; AVX1: # %bb.0: -; AVX1-NEXT: vpsrld $16, %xmm1, %xmm2 -; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1 ; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero -; AVX1-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3] ; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vmovddup {{.*#+}} xmm3 = [2,2] +; AVX1-NEXT: # xmm3 = mem[0,0] +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm1, %xmm4 ; AVX1-NEXT: vpaddq %xmm2, %xmm2, %xmm2 -; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm4 -; AVX1-NEXT: vpermilpd %ymm4, %ymm3, %ymm3 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3 +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm0[2,3,2,3] +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 +; AVX1-NEXT: vpermilpd %ymm1, %ymm4, %ymm2 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX1-NEXT: vpermilpd %ymm4, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm2, %xmm2 -; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 -; AVX1-NEXT: vblendvpd %ymm1, %ymm3, %ymm0, %ymm0 +; AVX1-NEXT: vpermilpd %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vblendvpd %ymm3, %ymm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: var_shuffle_v4i64_with_v16i8_indices: diff --git a/llvm/test/CodeGen/X86/vec_fcopysign.ll b/llvm/test/CodeGen/X86/vec_fcopysign.ll index 457210732396bdd20cc24012037298402f619276..b34b02c90796b3fd048bdf99698c586910ed20a0 100644 --- a/llvm/test/CodeGen/X86/vec_fcopysign.ll +++ b/llvm/test/CodeGen/X86/vec_fcopysign.ll @@ -159,8 +159,8 @@ define <8 x half> @fcopysign_v8f16(ptr %p0, ptr %p1) nounwind { ; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-AVX512-NEXT: vmovdqa (%ecx), %xmm1 -; X86-AVX512-NEXT: vpbroadcastw {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] -; X86-AVX512-NEXT: vpternlogq $202, (%eax), %xmm1, %xmm0 +; X86-AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [2147450879,2147450879,2147450879,2147450879] +; X86-AVX512-NEXT: vpternlogd $202, (%eax), %xmm1, %xmm0 ; X86-AVX512-NEXT: retl ; ; X64-SSE-LABEL: fcopysign_v8f16: @@ -193,8 +193,8 @@ define <8 x half> @fcopysign_v8f16(ptr %p0, ptr %p1) nounwind { ; X64-AVX512-LABEL: fcopysign_v8f16: ; X64-AVX512: # %bb.0: ; X64-AVX512-NEXT: vmovdqa (%rdi), %xmm1 -; X64-AVX512-NEXT: vpbroadcastq {{.*#+}} xmm0 = [9223231297218904063,9223231297218904063] -; X64-AVX512-NEXT: vpternlogq $202, (%rsi), %xmm1, %xmm0 +; X64-AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [2147450879,2147450879,2147450879,2147450879] +; X64-AVX512-NEXT: vpternlogd $202, (%rsi), %xmm1, %xmm0 ; X64-AVX512-NEXT: retq %a0 = load <8 x half>, ptr %p0, align 16 %a1 = load <8 x half>, ptr %p1, align 16 @@ -405,8 +405,8 @@ define <16 x half> @fcopysign_v16f16(ptr %p0, ptr %p1) nounwind { ; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-AVX512-NEXT: vmovdqu (%ecx), %ymm1 -; X86-AVX512-NEXT: vpbroadcastw {{.*#+}} ymm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] -; X86-AVX512-NEXT: vpternlogq $202, (%eax), %ymm1, %ymm0 +; X86-AVX512-NEXT: vpbroadcastd {{.*#+}} ymm0 = [2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879] +; X86-AVX512-NEXT: vpternlogd $202, (%eax), %ymm1, %ymm0 ; X86-AVX512-NEXT: retl ; ; X64-SSE-LABEL: fcopysign_v16f16: @@ -444,8 +444,8 @@ define <16 x half> @fcopysign_v16f16(ptr %p0, ptr %p1) nounwind { ; X64-AVX512-LABEL: fcopysign_v16f16: ; X64-AVX512: # %bb.0: ; X64-AVX512-NEXT: vmovdqu (%rdi), %ymm1 -; X64-AVX512-NEXT: vpbroadcastq {{.*#+}} ymm0 = [9223231297218904063,9223231297218904063,9223231297218904063,9223231297218904063] -; X64-AVX512-NEXT: vpternlogq $202, (%rsi), %ymm1, %ymm0 +; X64-AVX512-NEXT: vpbroadcastd {{.*#+}} ymm0 = [2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879] +; X64-AVX512-NEXT: vpternlogd $202, (%rsi), %ymm1, %ymm0 ; X64-AVX512-NEXT: retq %a0 = load <16 x half>, ptr %p0, align 16 %a1 = load <16 x half>, ptr %p1, align 16 @@ -691,34 +691,14 @@ define <32 x half> @fcopysign_v32f16(ptr %p0, ptr %p1) nounwind { ; X86-AVX2-NEXT: vpor %ymm2, %ymm1, %ymm1 ; X86-AVX2-NEXT: retl ; -; X86-AVX512VL-LABEL: fcopysign_v32f16: -; X86-AVX512VL: # %bb.0: -; X86-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-AVX512VL-NEXT: vmovdqu64 (%ecx), %zmm1 -; X86-AVX512VL-NEXT: vpbroadcastw {{.*#+}} ymm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] -; X86-AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0 -; X86-AVX512VL-NEXT: vpternlogq $202, (%eax), %zmm1, %zmm0 -; X86-AVX512VL-NEXT: retl -; -; X86-AVX512FP16-LABEL: fcopysign_v32f16: -; X86-AVX512FP16: # %bb.0: -; X86-AVX512FP16-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-AVX512FP16-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-AVX512FP16-NEXT: vmovdqu64 (%ecx), %zmm1 -; X86-AVX512FP16-NEXT: vpbroadcastw {{.*#+}} zmm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] -; X86-AVX512FP16-NEXT: vpternlogq $202, (%eax), %zmm1, %zmm0 -; X86-AVX512FP16-NEXT: retl -; -; X86-AVX512VLDQ-LABEL: fcopysign_v32f16: -; X86-AVX512VLDQ: # %bb.0: -; X86-AVX512VLDQ-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-AVX512VLDQ-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-AVX512VLDQ-NEXT: vmovdqu64 (%ecx), %zmm1 -; X86-AVX512VLDQ-NEXT: vpbroadcastw {{.*#+}} ymm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] -; X86-AVX512VLDQ-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0 -; X86-AVX512VLDQ-NEXT: vpternlogq $202, (%eax), %zmm1, %zmm0 -; X86-AVX512VLDQ-NEXT: retl +; X86-AVX512-LABEL: fcopysign_v32f16: +; X86-AVX512: # %bb.0: +; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-AVX512-NEXT: vmovdqu64 (%ecx), %zmm1 +; X86-AVX512-NEXT: vpbroadcastd {{.*#+}} zmm0 = [2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879] +; X86-AVX512-NEXT: vpternlogd $202, (%eax), %zmm1, %zmm0 +; X86-AVX512-NEXT: retl ; ; X64-SSE-LABEL: fcopysign_v32f16: ; X64-SSE: # %bb.0: @@ -769,8 +749,8 @@ define <32 x half> @fcopysign_v32f16(ptr %p0, ptr %p1) nounwind { ; X64-AVX512-LABEL: fcopysign_v32f16: ; X64-AVX512: # %bb.0: ; X64-AVX512-NEXT: vmovdqu64 (%rdi), %zmm1 -; X64-AVX512-NEXT: vpbroadcastq {{.*#+}} zmm0 = [9223231297218904063,9223231297218904063,9223231297218904063,9223231297218904063,9223231297218904063,9223231297218904063,9223231297218904063,9223231297218904063] -; X64-AVX512-NEXT: vpternlogq $202, (%rsi), %zmm1, %zmm0 +; X64-AVX512-NEXT: vpbroadcastd {{.*#+}} zmm0 = [2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879] +; X64-AVX512-NEXT: vpternlogd $202, (%rsi), %zmm1, %zmm0 ; X64-AVX512-NEXT: retq %a0 = load <32 x half>, ptr %p0, align 16 %a1 = load <32 x half>, ptr %p1, align 16 @@ -786,3 +766,6 @@ declare <32 x half> @llvm.copysign.v32f16(<32 x half>, <32 x half>) ; X64-AVX512VLDQ: {{.*}} ; X86: {{.*}} ; X86-AVX: {{.*}} +; X86-AVX512FP16: {{.*}} +; X86-AVX512VL: {{.*}} +; X86-AVX512VLDQ: {{.*}} diff --git a/llvm/test/CodeGen/X86/vec_zero_cse.ll b/llvm/test/CodeGen/X86/vec_zero_cse.ll index 21da191d8d69373436aec7561462a484e40adb60..d4357aeb2e1deaf2e3451e27cd506e01532d85ef 100644 --- a/llvm/test/CodeGen/X86/vec_zero_cse.ll +++ b/llvm/test/CodeGen/X86/vec_zero_cse.ll @@ -16,7 +16,7 @@ define void @test1() { ; X86-NEXT: movl $0, M1+4 ; X86-NEXT: movl $0, M1 ; X86-NEXT: xorps %xmm0, %xmm0 -; X86-NEXT: movlps %xmm0, M2 +; X86-NEXT: movsd %xmm0, M2 ; X86-NEXT: retl ; ; X64-LABEL: test1: diff --git a/llvm/test/CodeGen/X86/vector-fshl-128.ll b/llvm/test/CodeGen/X86/vector-fshl-128.ll index 2d0e92a54846b08620091b86589ed2a32c76a709..4f100cd3e05309baae509a8f56c13aa840bac680 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-128.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-128.ll @@ -2409,7 +2409,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512F-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq @@ -2418,14 +2418,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VL-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm2, %xmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v16i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512BW-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq @@ -2434,7 +2434,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VBMI2-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512VBMI2-NEXT: vzeroupper ; AVX512VBMI2-NEXT: retq @@ -2443,14 +2443,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VLBW-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm2, %xmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i8: ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm2, %xmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0 ; AVX512VLVBMI2-NEXT: retq ; ; XOP-LABEL: splatconstant_funnnel_v16i8: diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll index 43d426c326da7d94ebf0ec8628cc273ca3dd078c..05be4e1ee928e43115d18811694a4646f3f0f178 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-256.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll @@ -2303,7 +2303,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512F-NEXT: retq ; @@ -2311,14 +2311,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v32i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512BW-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; @@ -2326,7 +2326,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VBMI2-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512VBMI2-NEXT: retq ; @@ -2334,14 +2334,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VLBW-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; AVX512VLBW-NEXT: retq ; ; AVX10-LABEL: splatconstant_funnnel_v32i8: ; AVX10: # %bb.0: ; AVX10-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX10-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX10-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; AVX10-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; AVX10-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8: diff --git a/llvm/test/CodeGen/X86/vector-fshl-512.ll b/llvm/test/CodeGen/X86/vector-fshl-512.ll index c246aaf61dbcfe9ae9c78ba715800e2678f172d9..0724d879f557ddd7e8d9893ce99508a1e259429d 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-512.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-512.ll @@ -1124,7 +1124,7 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi ; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm1 ; AVX512F-NEXT: vpsrlw $4, %ymm1, %ymm1 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: splatconstant_funnnel_v64i8: @@ -1137,35 +1137,35 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi ; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm1 ; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm1 ; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v64i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512BW-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512BW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512VBMI2-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VBMI2-NEXT: retq ; ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512VLBW-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VLVBMI2-NEXT: retq %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> ) ret <64 x i8> %res diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll index 37d4f3b3dff54dfdee759358616c8b6725f11f01..9ddd171b4db6900963f774ad17ef35e8e2443e64 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll @@ -1851,7 +1851,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512F-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq @@ -1860,14 +1860,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v16i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512BW-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq @@ -1876,14 +1876,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VLBW-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512VBMI2-NEXT: vzeroupper ; AVX512VBMI2-NEXT: retq @@ -1892,7 +1892,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VLVBMI2-NEXT: retq ; ; XOP-LABEL: splatconstant_funnnel_v16i8: diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll index 8762072f3e8f6cc9ac22d68f53c2bee3fa6d4876..58719e6bd8e0c3ce0e274edae32a6af32df07ca4 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll @@ -447,12 +447,12 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm3 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm3 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm3 ; AVX512F-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512F-NEXT: vpsllw $2, %ymm0, %ymm2 ; AVX512F-NEXT: vpsrlw $6, %ymm0, %ymm3 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm3 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm3 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm2 @@ -467,12 +467,12 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm3 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3 ; AVX512VL-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsllw $2, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $6, %ymm0, %ymm3 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm3 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $7, %ymm0, %ymm2 @@ -1652,7 +1652,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512F-NEXT: retq ; @@ -1660,14 +1660,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v32i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512BW-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; @@ -1675,14 +1675,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VLBW-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512VBMI2-NEXT: retq ; @@ -1690,7 +1690,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8: diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll index 0a473dd1ed82419f3b4c0b503b096670965612d7..edfa56a70d59e92db03c8594ee4f051d3ec0f40a 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll @@ -131,15 +131,15 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind { ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm3 ; AVX512F-NEXT: vpsllw $4, %ymm2, %ymm4 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm5 = [17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520] -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm5, %zmm4 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm5 = [4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160] +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm5, %zmm4 ; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm3 ; AVX512F-NEXT: vpsllw $5, %ymm3, %ymm3 ; AVX512F-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512F-NEXT: vpsrlw $6, %ymm2, %ymm4 ; AVX512F-NEXT: vpsllw $2, %ymm2, %ymm6 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm7 = [18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596] -; AVX512F-NEXT: vpternlogq $226, %zmm4, %zmm7, %zmm6 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm7 = [4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268] +; AVX512F-NEXT: vpternlogd $226, %zmm4, %zmm7, %zmm6 ; AVX512F-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512F-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512F-NEXT: vpsrlw $7, %ymm2, %ymm4 @@ -151,12 +151,12 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind { ; AVX512F-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm3 ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm4 -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm5, %zmm4 +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm5, %zmm4 ; AVX512F-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $6, %ymm0, %ymm3 ; AVX512F-NEXT: vpsllw $2, %ymm0, %ymm4 -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm7, %zmm4 +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm7, %zmm4 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm3 @@ -173,15 +173,15 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind { ; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm3 ; AVX512VL-NEXT: vpsllw $4, %ymm2, %ymm4 -; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm5 = [17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520] -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm5, %ymm4 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm5 = [4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160] +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm5, %ymm4 ; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm3 ; AVX512VL-NEXT: vpsllw $5, %ymm3, %ymm3 ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsrlw $6, %ymm2, %ymm4 ; AVX512VL-NEXT: vpsllw $2, %ymm2, %ymm6 -; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm7 = [18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596] -; AVX512VL-NEXT: vpternlogq $226, %ymm4, %ymm7, %ymm6 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm7 = [4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268] +; AVX512VL-NEXT: vpternlogd $226, %ymm4, %ymm7, %ymm6 ; AVX512VL-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsrlw $7, %ymm2, %ymm4 @@ -192,12 +192,12 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind { ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3 ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm4 -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm5, %ymm4 +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm5, %ymm4 ; AVX512VL-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $6, %ymm0, %ymm3 ; AVX512VL-NEXT: vpsllw $2, %ymm0, %ymm4 -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm7, %ymm4 +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm7, %ymm4 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $7, %ymm0, %ymm3 @@ -721,7 +721,7 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind { ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: splatconstant_funnnel_v64i8: @@ -733,35 +733,35 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind { ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v64i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: retq ; ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLBW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: retq ; ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLVBMI2-NEXT: retq %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> ) ret <64 x i8> %res diff --git a/llvm/test/CodeGen/X86/vector-fshr-128.ll b/llvm/test/CodeGen/X86/vector-fshr-128.ll index 809735a88f20801cb5fbc31c12340389280eb334..eb2df0dcda98a354dab09f94ea1ce077d6880be7 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-128.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-128.ll @@ -2403,7 +2403,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512F-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq @@ -2412,14 +2412,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VL-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm2, %xmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v16i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512BW-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq @@ -2428,7 +2428,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VBMI2-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512VBMI2-NEXT: vzeroupper ; AVX512VBMI2-NEXT: retq @@ -2437,14 +2437,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VLBW-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm2, %xmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i8: ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm2, %xmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0 ; AVX512VLVBMI2-NEXT: retq ; ; XOP-LABEL: splatconstant_funnnel_v16i8: diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll index fa41a10adb5e839b0c009243b980157c545341d5..1a6ecea596563ed3c2ef8a0150690fb01d509dc4 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-256.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll @@ -2104,7 +2104,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512F-NEXT: retq ; @@ -2112,14 +2112,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v32i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512BW-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; @@ -2127,7 +2127,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VBMI2-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512VBMI2-NEXT: retq ; @@ -2135,14 +2135,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VLBW-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; AVX512VLBW-NEXT: retq ; ; AVX10-LABEL: splatconstant_funnnel_v32i8: ; AVX10: # %bb.0: ; AVX10-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX10-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX10-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; AVX10-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; AVX10-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8: diff --git a/llvm/test/CodeGen/X86/vector-fshr-512.ll b/llvm/test/CodeGen/X86/vector-fshr-512.ll index b1fee9d1b0b3986c5238454d100e58371ccdafe7..1c6646152ab1f7dd279077322d3df803c16ab41c 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-512.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-512.ll @@ -1180,7 +1180,7 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi ; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm1 ; AVX512F-NEXT: vpsrlw $4, %ymm1, %ymm1 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: splatconstant_funnnel_v64i8: @@ -1193,35 +1193,35 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi ; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm1 ; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm1 ; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v64i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512BW-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512BW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512VBMI2-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VBMI2-NEXT: retq ; ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512VLBW-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VLVBMI2-NEXT: retq %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> ) ret <64 x i8> %res diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll index 9c5fe49e7d0ca4181a5a84073a89c9cbbb611f71..402eb73e18101bde0967ba031b2b018a6d7a8e15 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll @@ -1915,7 +1915,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512F-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq @@ -1924,14 +1924,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v16i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512BW-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq @@ -1940,14 +1940,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VLBW-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512VBMI2-NEXT: vzeroupper ; AVX512VBMI2-NEXT: retq @@ -1956,7 +1956,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VLVBMI2-NEXT: retq ; ; XOP-LABEL: splatconstant_funnnel_v16i8: diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll index e4867ba09696852bf4a673c6c443eb7bd0fd1142..bb311468ce913cb70f96cc77caaaa6a4865c8f49 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll @@ -473,17 +473,17 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm2 ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm3 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm3 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm3 ; AVX512F-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $2, %ymm0, %ymm2 ; AVX512F-NEXT: vpsllw $6, %ymm0, %ymm3 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm3 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm3 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $1, %ymm0, %ymm2 ; AVX512F-NEXT: vpsllw $7, %ymm0, %ymm3 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm3 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm3 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512F-NEXT: retq @@ -492,17 +492,17 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm3 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm3 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3 ; AVX512VL-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $2, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsllw $6, %ymm0, %ymm3 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm3 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $1, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsllw $7, %ymm0, %ymm3 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm3 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512VL-NEXT: retq @@ -1703,7 +1703,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512F-NEXT: retq ; @@ -1711,14 +1711,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v32i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512BW-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; @@ -1726,14 +1726,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VLBW-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512VBMI2-NEXT: retq ; @@ -1741,7 +1741,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8: diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll index 3c17bf2f6b9a625431f939f12a4495b5500ab465..4364c047dfdebfcc7e34a2f7fde3f4753f90fd27 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll @@ -131,36 +131,36 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind { ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2 ; AVX512F-NEXT: vpsllw $4, %ymm2, %ymm3 ; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm4 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm5 = [1085102592571150095,1085102592571150095,1085102592571150095,1085102592571150095,1085102592571150095,1085102592571150095,1085102592571150095,1085102592571150095] -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm5, %zmm4 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm5 = [252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135] +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm5, %zmm4 ; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm3 ; AVX512F-NEXT: vpsllw $5, %ymm3, %ymm3 ; AVX512F-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512F-NEXT: vpsllw $6, %ymm2, %ymm4 ; AVX512F-NEXT: vpsrlw $2, %ymm2, %ymm6 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm7 = [4557430888798830399,4557430888798830399,4557430888798830399,4557430888798830399,4557430888798830399,4557430888798830399,4557430888798830399,4557430888798830399] -; AVX512F-NEXT: vpternlogq $226, %zmm4, %zmm7, %zmm6 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm7 = [1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567] +; AVX512F-NEXT: vpternlogd $226, %zmm4, %zmm7, %zmm6 ; AVX512F-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512F-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512F-NEXT: vpsllw $7, %ymm2, %ymm4 ; AVX512F-NEXT: vpsrlw $1, %ymm2, %ymm6 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm8 = [9187201950435737471,9187201950435737471,9187201950435737471,9187201950435737471,9187201950435737471,9187201950435737471,9187201950435737471,9187201950435737471] -; AVX512F-NEXT: vpternlogq $226, %zmm4, %zmm8, %zmm6 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm8 = [2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143] +; AVX512F-NEXT: vpternlogd $226, %zmm4, %zmm8, %zmm6 ; AVX512F-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512F-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm3 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm4 -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm5, %zmm4 +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm5, %zmm4 ; AVX512F-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsllw $6, %ymm0, %ymm3 ; AVX512F-NEXT: vpsrlw $2, %ymm0, %ymm4 -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm7, %zmm4 +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm7, %zmm4 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsllw $7, %ymm0, %ymm3 ; AVX512F-NEXT: vpsrlw $1, %ymm0, %ymm4 -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm8, %zmm4 +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm8, %zmm4 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 @@ -171,36 +171,36 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind { ; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2 ; AVX512VL-NEXT: vpsllw $4, %ymm2, %ymm3 ; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm4 -; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm5 = [1085102592571150095,1085102592571150095,1085102592571150095,1085102592571150095] -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm5, %ymm4 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm5 = [252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135] +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm5, %ymm4 ; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm3 ; AVX512VL-NEXT: vpsllw $5, %ymm3, %ymm3 ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsllw $6, %ymm2, %ymm4 ; AVX512VL-NEXT: vpsrlw $2, %ymm2, %ymm6 -; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm7 = [4557430888798830399,4557430888798830399,4557430888798830399,4557430888798830399] -; AVX512VL-NEXT: vpternlogq $226, %ymm4, %ymm7, %ymm6 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm7 = [1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567] +; AVX512VL-NEXT: vpternlogd $226, %ymm4, %ymm7, %ymm6 ; AVX512VL-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsllw $7, %ymm2, %ymm4 ; AVX512VL-NEXT: vpsrlw $1, %ymm2, %ymm6 -; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm8 = [9187201950435737471,9187201950435737471,9187201950435737471,9187201950435737471] -; AVX512VL-NEXT: vpternlogq $226, %ymm4, %ymm8, %ymm6 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm8 = [2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143] +; AVX512VL-NEXT: vpternlogd $226, %ymm4, %ymm8, %ymm6 ; AVX512VL-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm3 ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm4 -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm5, %ymm4 +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm5, %ymm4 ; AVX512VL-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsllw $6, %ymm0, %ymm3 ; AVX512VL-NEXT: vpsrlw $2, %ymm0, %ymm4 -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm7, %ymm4 +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm7, %ymm4 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsllw $7, %ymm0, %ymm3 ; AVX512VL-NEXT: vpsrlw $1, %ymm0, %ymm4 -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm8, %ymm4 +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm8, %ymm4 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 @@ -721,7 +721,7 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind { ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: splatconstant_funnnel_v64i8: @@ -733,35 +733,35 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind { ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v64i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: retq ; ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLBW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: retq ; ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLVBMI2-NEXT: retq %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> ) ret <64 x i8> %res diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-4.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-4.ll index c9cbb0994810f32364644bd55fc174fd5b0ba37f..e2195f1fc25a1345717bfbfed80ada587d6bf1ea 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-4.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-4.ll @@ -156,64 +156,62 @@ define void @load_i8_stride4_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr define void @load_i8_stride4_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind { ; SSE-LABEL: load_i8_stride4_vf8: ; SSE: # %bb.0: -; SSE-NEXT: movdqa (%rdi), %xmm1 +; SSE-NEXT: movdqa (%rdi), %xmm3 ; SSE-NEXT: movdqa 16(%rdi), %xmm4 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,0,255,0,255,0,255,0] +; SSE-NEXT: movdqa %xmm4, %xmm1 ; SSE-NEXT: movdqa %xmm4, %xmm2 -; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: pand %xmm1, %xmm0 -; SSE-NEXT: packuswb %xmm2, %xmm0 +; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm4[3,1,2,3,4,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm4 +; SSE-NEXT: pand %xmm3, %xmm0 +; SSE-NEXT: packuswb %xmm4, %xmm0 ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pxor %xmm7, %xmm7 -; SSE-NEXT: movdqa %xmm4, %xmm2 -; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm7[8],xmm2[9],xmm7[9],xmm2[10],xmm7[10],xmm2[11],xmm7[11],xmm2[12],xmm7[12],xmm2[13],xmm7[13],xmm2[14],xmm7[14],xmm2[15],xmm7[15] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm3[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm4, %xmm3 -; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3],xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm6[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1] -; SSE-NEXT: movdqa %xmm1, %xmm5 -; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm6[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm1, %xmm6 -; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7] -; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1] -; SSE-NEXT: packuswb %xmm8, %xmm7 -; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,3,2,3] -; SSE-NEXT: movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm8, %xmm4 -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm8, %xmm1 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm4, %xmm1 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] +; SSE-NEXT: pxor %xmm6, %xmm6 +; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7] +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm2[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1] +; SSE-NEXT: movdqa %xmm3, %xmm4 +; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm6[8],xmm3[9],xmm6[9],xmm3[10],xmm6[10],xmm3[11],xmm6[11],xmm3[12],xmm6[12],xmm3[13],xmm6[13],xmm3[14],xmm6[14],xmm3[15],xmm6[15] +; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm3[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1] +; SSE-NEXT: packuswb %xmm7, %xmm6 +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,3,2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,5,6,7] +; SSE-NEXT: movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm7, %xmm5 +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm7, %xmm8 +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm8[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm5, %xmm7 +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm7[0,3,2,3] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm5[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm6[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1] -; SSE-NEXT: packuswb %xmm3, %xmm4 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,3,2,3] +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm4[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1] +; SSE-NEXT: packuswb %xmm2, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,3,2,3] ; SSE-NEXT: movq %xmm0, (%rsi) -; SSE-NEXT: movq %xmm7, (%rdx) -; SSE-NEXT: movq %xmm1, (%rcx) -; SSE-NEXT: movq %xmm2, (%r8) +; SSE-NEXT: movq %xmm6, (%rdx) +; SSE-NEXT: movq %xmm5, (%rcx) +; SSE-NEXT: movq %xmm1, (%r8) ; SSE-NEXT: retq ; ; AVX1-ONLY-LABEL: load_i8_stride4_vf8: @@ -296,111 +294,111 @@ define void @load_i8_stride4_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-LABEL: load_i8_stride4_vf16: ; SSE: # %bb.0: ; SSE-NEXT: movdqa (%rdi), %xmm1 -; SSE-NEXT: movdqa 16(%rdi), %xmm2 -; SSE-NEXT: movdqa 32(%rdi), %xmm8 -; SSE-NEXT: movdqa 48(%rdi), %xmm13 -; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,0,255,0,255,0,255,0] -; SSE-NEXT: movdqa %xmm13, %xmm3 -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: movdqa %xmm8, %xmm4 -; SSE-NEXT: pand %xmm0, %xmm4 -; SSE-NEXT: packuswb %xmm3, %xmm4 -; SSE-NEXT: movdqa %xmm2, %xmm3 -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: pand %xmm1, %xmm0 -; SSE-NEXT: packuswb %xmm3, %xmm0 -; SSE-NEXT: packuswb %xmm4, %xmm0 -; SSE-NEXT: pxor %xmm10, %xmm10 -; SSE-NEXT: movdqa %xmm13, %xmm3 -; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm10[8],xmm3[9],xmm10[9],xmm3[10],xmm10[10],xmm3[11],xmm10[11],xmm3[12],xmm10[12],xmm3[13],xmm10[13],xmm3[14],xmm10[14],xmm3[15],xmm10[15] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm13, %xmm4 -; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3],xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm6[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1] -; SSE-NEXT: movdqa %xmm8, %xmm5 -; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm10[8],xmm5[9],xmm10[9],xmm5[10],xmm10[10],xmm5[11],xmm10[11],xmm5[12],xmm10[12],xmm5[13],xmm10[13],xmm5[14],xmm10[14],xmm5[15],xmm10[15] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm6[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm8, %xmm6 -; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm10[0],xmm6[1],xmm10[1],xmm6[2],xmm10[2],xmm6[3],xmm10[3],xmm6[4],xmm10[4],xmm6[5],xmm10[5],xmm6[6],xmm10[6],xmm6[7],xmm10[7] -; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm6[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm11[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm9[0],xmm14[1],xmm9[1] -; SSE-NEXT: packuswb %xmm7, %xmm14 -; SSE-NEXT: movdqa %xmm2, %xmm7 -; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm10[8],xmm7[9],xmm10[9],xmm7[10],xmm10[10],xmm7[11],xmm10[11],xmm7[12],xmm10[12],xmm7[13],xmm10[13],xmm7[14],xmm10[14],xmm7[15],xmm10[15] -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm7[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm9[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm2, %xmm9 -; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3],xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7] -; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm9[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm12[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm11[0],xmm15[1],xmm11[1] -; SSE-NEXT: movdqa %xmm1, %xmm12 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm1, %xmm11 -; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm10[8],xmm11[9],xmm10[9],xmm11[10],xmm10[10],xmm11[11],xmm10[11],xmm11[12],xmm10[12],xmm11[13],xmm10[13],xmm11[14],xmm10[14],xmm11[15],xmm10[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3],xmm12[4],xmm10[4],xmm12[5],xmm10[5],xmm12[6],xmm10[6],xmm12[7],xmm10[7] -; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm11[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm10[1,3,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm12[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm1[0],xmm10[1],xmm1[1] -; SSE-NEXT: packuswb %xmm15, %xmm10 -; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[0,3],xmm14[0,3] -; SSE-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm1, %xmm13 -; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm13[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm13[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm1, %xmm8 -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm13, %xmm8 +; SSE-NEXT: movdqa 16(%rdi), %xmm3 +; SSE-NEXT: movdqa 32(%rdi), %xmm9 +; SSE-NEXT: movdqa 48(%rdi), %xmm10 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0] +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: movdqa %xmm10, %xmm4 +; SSE-NEXT: movdqa %xmm10, %xmm7 +; SSE-NEXT: pand %xmm2, %xmm7 +; SSE-NEXT: movdqa %xmm9, %xmm5 +; SSE-NEXT: movdqa %xmm9, %xmm6 +; SSE-NEXT: movdqa %xmm9, %xmm11 +; SSE-NEXT: pand %xmm2, %xmm11 +; SSE-NEXT: packuswb %xmm7, %xmm11 +; SSE-NEXT: movdqa %xmm3, %xmm7 +; SSE-NEXT: movdqa %xmm3, %xmm8 +; SSE-NEXT: movdqa %xmm3, %xmm12 +; SSE-NEXT: pand %xmm2, %xmm12 ; SSE-NEXT: pand %xmm1, %xmm2 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload -; SSE-NEXT: pand %xmm1, %xmm13 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,7,6,5,4] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm13[3,1,2,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm12, %xmm2 +; SSE-NEXT: packuswb %xmm11, %xmm2 +; SSE-NEXT: pxor %xmm11, %xmm11 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm11[8],xmm0[9],xmm11[9],xmm0[10],xmm11[10],xmm0[11],xmm11[11],xmm0[12],xmm11[12],xmm0[13],xmm11[13],xmm0[14],xmm11[14],xmm0[15],xmm11[15] +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm0[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm12[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm11[0],xmm4[1],xmm11[1],xmm4[2],xmm11[2],xmm4[3],xmm11[3],xmm4[4],xmm11[4],xmm4[5],xmm11[5],xmm4[6],xmm11[6],xmm4[7],xmm11[7] +; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm4[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm13[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm12[0],xmm13[1],xmm12[1] +; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm11[8],xmm5[9],xmm11[9],xmm5[10],xmm11[10],xmm5[11],xmm11[11],xmm5[12],xmm11[12],xmm5[13],xmm11[13],xmm5[14],xmm11[14],xmm5[15],xmm11[15] +; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm5[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm11[0],xmm6[1],xmm11[1],xmm6[2],xmm11[2],xmm6[3],xmm11[3],xmm6[4],xmm11[4],xmm6[5],xmm11[5],xmm6[6],xmm11[6],xmm6[7],xmm11[7] +; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm6[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm14[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm12[0],xmm14[1],xmm12[1] +; SSE-NEXT: packuswb %xmm13, %xmm14 +; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm11[8],xmm7[9],xmm11[9],xmm7[10],xmm11[10],xmm7[11],xmm11[11],xmm7[12],xmm11[12],xmm7[13],xmm11[13],xmm7[14],xmm11[14],xmm7[15],xmm11[15] +; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm7[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm12[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm11[0],xmm8[1],xmm11[1],xmm8[2],xmm11[2],xmm8[3],xmm11[3],xmm8[4],xmm11[4],xmm8[5],xmm11[5],xmm8[6],xmm11[6],xmm8[7],xmm11[7] +; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm8[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm13[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm12[0],xmm15[1],xmm12[1] +; SSE-NEXT: movdqa %xmm1, %xmm12 +; SSE-NEXT: movdqa %xmm1, %xmm13 +; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm11[8],xmm13[9],xmm11[9],xmm13[10],xmm11[10],xmm13[11],xmm11[11],xmm13[12],xmm11[12],xmm13[13],xmm11[13],xmm13[14],xmm11[14],xmm13[15],xmm11[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3],xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7] +; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm13[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm11[1,3,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm12[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm0[0],xmm11[1],xmm0[1] +; SSE-NEXT: packuswb %xmm15, %xmm11 +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[3,1,2,3,4,5,6,7] +; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[0,3],xmm14[0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,7] +; SSE-NEXT: movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm10, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm10, %xmm9 +; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm0, %xmm9 +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm10, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm1 -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm8[0,3] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] +; SSE-NEXT: packuswb %xmm0, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm9[0,3] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; SSE-NEXT: # xmm0 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm4[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm5[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm5[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm6[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] ; SSE-NEXT: packuswb %xmm3, %xmm4 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm7[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm9[3,1,2,3] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm7[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm8[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm11[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm13[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm12[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1] +; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1] ; SSE-NEXT: packuswb %xmm3, %xmm5 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,3],xmm4[0,3] -; SSE-NEXT: movdqa %xmm0, (%rsi) -; SSE-NEXT: movaps %xmm10, (%rdx) +; SSE-NEXT: movdqa %xmm2, (%rsi) +; SSE-NEXT: movaps %xmm11, (%rdx) ; SSE-NEXT: movaps %xmm1, (%rcx) ; SSE-NEXT: movaps %xmm5, (%r8) ; SSE-NEXT: retq @@ -530,249 +528,247 @@ define void @load_i8_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE: # %bb.0: ; SSE-NEXT: subq $136, %rsp ; SSE-NEXT: movdqa 64(%rdi), %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 80(%rdi), %xmm5 -; SSE-NEXT: movdqa 96(%rdi), %xmm15 -; SSE-NEXT: movdqa 112(%rdi), %xmm14 -; SSE-NEXT: movdqa (%rdi), %xmm12 -; SSE-NEXT: movdqa 16(%rdi), %xmm11 -; SSE-NEXT: movdqa 32(%rdi), %xmm7 -; SSE-NEXT: movdqa 48(%rdi), %xmm3 -; SSE-NEXT: movdqa {{.*#+}} xmm6 = [255,0,255,0,255,0,255,0] -; SSE-NEXT: movdqa %xmm3, %xmm0 -; SSE-NEXT: pand %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: pand %xmm6, %xmm1 -; SSE-NEXT: packuswb %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm11, %xmm0 -; SSE-NEXT: pand %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm12, %xmm2 -; SSE-NEXT: pand %xmm6, %xmm2 -; SSE-NEXT: packuswb %xmm0, %xmm2 -; SSE-NEXT: packuswb %xmm1, %xmm2 +; SSE-NEXT: movdqa 80(%rdi), %xmm9 +; SSE-NEXT: movdqa 96(%rdi), %xmm11 +; SSE-NEXT: movdqa 112(%rdi), %xmm12 +; SSE-NEXT: movdqa (%rdi), %xmm13 +; SSE-NEXT: movdqa 16(%rdi), %xmm14 +; SSE-NEXT: movdqa 32(%rdi), %xmm2 +; SSE-NEXT: movdqa 48(%rdi), %xmm10 +; SSE-NEXT: movdqa {{.*#+}} xmm8 = [255,0,255,0,255,0,255,0] +; SSE-NEXT: movdqa %xmm10, %xmm3 +; SSE-NEXT: movdqa %xmm10, %xmm5 +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: pand %xmm8, %xmm0 +; SSE-NEXT: movdqa %xmm2, %xmm6 +; SSE-NEXT: movdqa %xmm2, %xmm1 +; SSE-NEXT: movdqa %xmm2, %xmm15 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm8, %xmm1 +; SSE-NEXT: packuswb %xmm0, %xmm1 ; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: pand %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm15, %xmm1 -; SSE-NEXT: pand %xmm6, %xmm1 +; SSE-NEXT: pand %xmm8, %xmm0 +; SSE-NEXT: movdqa %xmm13, %xmm7 +; SSE-NEXT: pand %xmm8, %xmm7 +; SSE-NEXT: packuswb %xmm0, %xmm7 +; SSE-NEXT: packuswb %xmm1, %xmm7 +; SSE-NEXT: movdqa %xmm12, %xmm0 +; SSE-NEXT: pand %xmm8, %xmm0 +; SSE-NEXT: movdqa %xmm11, %xmm1 +; SSE-NEXT: pand %xmm8, %xmm1 ; SSE-NEXT: packuswb %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm5, %xmm2 -; SSE-NEXT: pand %xmm6, %xmm0 -; SSE-NEXT: pand %xmm4, %xmm6 -; SSE-NEXT: packuswb %xmm0, %xmm6 -; SSE-NEXT: packuswb %xmm1, %xmm6 -; SSE-NEXT: pxor %xmm4, %xmm4 -; SSE-NEXT: movdqa %xmm3, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SSE-NEXT: movdqa %xmm9, %xmm0 +; SSE-NEXT: pand %xmm8, %xmm0 +; SSE-NEXT: pand %xmm4, %xmm8 +; SSE-NEXT: packuswb %xmm0, %xmm8 +; SSE-NEXT: packuswb %xmm1, %xmm8 +; SSE-NEXT: pxor %xmm2, %xmm2 +; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15] +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm3, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7] +; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15] +; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm15, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7] ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1] -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1] -; SSE-NEXT: packuswb %xmm5, %xmm8 -; SSE-NEXT: movdqa %xmm11, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] +; SSE-NEXT: packuswb %xmm3, %xmm5 +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm14, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm11, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; SSE-NEXT: movdqa %xmm14, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm5[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1] -; SSE-NEXT: movdqa %xmm12, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: movdqa %xmm13, %xmm14 +; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm13, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] ; SSE-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm12, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm5[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm0[0],xmm13[1],xmm0[1] -; SSE-NEXT: packuswb %xmm9, %xmm13 -; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[0,3],xmm8[0,3] -; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm2[0],xmm14[1],xmm2[1],xmm14[2],xmm2[2],xmm14[3],xmm2[3],xmm14[4],xmm2[4],xmm14[5],xmm2[5],xmm14[6],xmm2[6],xmm14[7],xmm2[7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm14[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm6[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; SSE-NEXT: packuswb %xmm3, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm5[0,3] +; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm12, %xmm15 +; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm12, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm14, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1] -; SSE-NEXT: movdqa %xmm15, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm15, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1] -; SSE-NEXT: packuswb %xmm8, %xmm0 -; SSE-NEXT: movdqa %xmm2, %xmm1 -; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm2, %xmm10 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm9[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload -; SSE-NEXT: movdqa %xmm5, %xmm9 -; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15] -; SSE-NEXT: movdqa %xmm5, %xmm8 -; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3],xmm8[4],xmm4[4],xmm8[5],xmm4[5],xmm8[6],xmm4[6],xmm8[7],xmm4[7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm9[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm4[1,3,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm8[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm2[0],xmm15[1],xmm2[1],xmm15[2],xmm2[2],xmm15[3],xmm2[3],xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm15[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: movdqa %xmm11, %xmm12 +; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm11, %xmm13 +; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm2[8],xmm13[9],xmm2[9],xmm13[10],xmm2[10],xmm13[11],xmm2[11],xmm13[12],xmm2[12],xmm13[13],xmm2[13],xmm13[14],xmm2[14],xmm13[15],xmm2[15] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm0[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm2[0],xmm12[1],xmm2[1],xmm12[2],xmm2[2],xmm12[3],xmm2[3],xmm12[4],xmm2[4],xmm12[5],xmm2[5],xmm12[6],xmm2[6],xmm12[7],xmm2[7] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm12[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm5[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1] +; SSE-NEXT: packuswb %xmm3, %xmm0 +; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm9, %xmm11 +; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm2[8],xmm11[9],xmm2[9],xmm11[10],xmm2[10],xmm11[11],xmm2[11],xmm11[12],xmm2[12],xmm11[13],xmm2[13],xmm11[14],xmm2[14],xmm11[15],xmm2[15] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm11[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3],xmm9[4],xmm2[4],xmm9[5],xmm2[5],xmm9[6],xmm2[6],xmm9[7],xmm2[7] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm9[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm5[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] +; SSE-NEXT: movdqa %xmm4, %xmm5 +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm4, %xmm6 +; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm6[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm3[1,3,2,3,4,5,6,7] ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1] ; SSE-NEXT: packuswb %xmm1, %xmm4 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,3],xmm0[0,3] -; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,7,5,6,7] +; SSE-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm3, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm3, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm2, %xmm1 +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload +; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload +; SSE-NEXT: # xmm10 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm3, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm3, %xmm10 +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm2, %xmm10 +; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[0,3],xmm1[0,3] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload +; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm3, %xmm1 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm0, %xmm7 -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm7[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm3, %xmm2 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,0,3,2,4,5,6,7] ; SSE-NEXT: packuswb %xmm1, %xmm2 -; SSE-NEXT: pand %xmm0, %xmm11 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm11[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm1[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm0, %xmm12 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm3, %xmm1 -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm2[0,3] -; SSE-NEXT: pand %xmm0, %xmm14 -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm14[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm0, %xmm15 -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm15[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm3[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm7 -; SSE-NEXT: pand %xmm0, %xmm10 -; SSE-NEXT: pand %xmm0, %xmm5 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[3,1,2,3,4,5,6,7] +; SSE-NEXT: pand %xmm3, %xmm1 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm5[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm2[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm0, %xmm3 -; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm7[0,3] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload -; SSE-NEXT: # xmm2 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload -; SSE-NEXT: # xmm7 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1] -; SSE-NEXT: packuswb %xmm2, %xmm7 -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7] +; SSE-NEXT: pand %xmm3, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm1, %xmm0 +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,3],xmm2[0,3] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload ; SSE-NEXT: # xmm2 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] -; SSE-NEXT: pshufd $231, (%rsp), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm0[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm11[0],xmm0[1],xmm11[1] -; SSE-NEXT: packuswb %xmm2, %xmm0 -; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,3],xmm7[0,3] +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1] +; SSE-NEXT: packuswb %xmm2, %xmm3 +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload ; SSE-NEXT: # xmm2 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload -; SSE-NEXT: # xmm7 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload -; SSE-NEXT: # xmm2 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload -; SSE-NEXT: # xmm11 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1] -; SSE-NEXT: packuswb %xmm7, %xmm11 -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload -; SSE-NEXT: # xmm2 = mem[3,1,2,3] +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: pshufd $231, (%rsp), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm14[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm14[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm1[0],xmm14[1],xmm1[1] +; SSE-NEXT: packuswb %xmm2, %xmm14 +; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[0,3],xmm3[0,3] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm15[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload -; SSE-NEXT: # xmm7 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1] +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm13[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm12[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1] +; SSE-NEXT: packuswb %xmm2, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm11[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm9[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm2[0],xmm8[1],xmm2[1] -; SSE-NEXT: packuswb %xmm7, %xmm8 -; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[0,3],xmm11[0,3] -; SSE-NEXT: movdqa %xmm6, 16(%rsi) -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload -; SSE-NEXT: movaps %xmm2, (%rsi) +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1] +; SSE-NEXT: packuswb %xmm2, %xmm5 +; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,3],xmm3[0,3] +; SSE-NEXT: movdqa %xmm8, 16(%rsi) +; SSE-NEXT: movdqa %xmm7, (%rsi) ; SSE-NEXT: movaps %xmm4, 16(%rdx) -; SSE-NEXT: movaps %xmm13, (%rdx) -; SSE-NEXT: movaps %xmm3, 16(%rcx) -; SSE-NEXT: movaps %xmm1, (%rcx) -; SSE-NEXT: movaps %xmm8, 16(%r8) -; SSE-NEXT: movaps %xmm0, (%r8) +; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; SSE-NEXT: movaps %xmm1, (%rdx) +; SSE-NEXT: movaps %xmm0, 16(%rcx) +; SSE-NEXT: movaps %xmm10, (%rcx) +; SSE-NEXT: movaps %xmm5, 16(%r8) +; SSE-NEXT: movaps %xmm14, (%r8) ; SSE-NEXT: addq $136, %rsp ; SSE-NEXT: retq ; @@ -1023,522 +1019,518 @@ define void @load_i8_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr define void @load_i8_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind { ; SSE-LABEL: load_i8_stride4_vf64: ; SSE: # %bb.0: -; SSE-NEXT: subq $664, %rsp # imm = 0x298 +; SSE-NEXT: subq $600, %rsp # imm = 0x258 ; SSE-NEXT: movdqa 16(%rdi), %xmm8 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 32(%rdi), %xmm6 +; SSE-NEXT: movdqa 32(%rdi), %xmm15 ; SSE-NEXT: movdqa 48(%rdi), %xmm14 -; SSE-NEXT: movdqa 128(%rdi), %xmm15 -; SSE-NEXT: movdqa 144(%rdi), %xmm10 -; SSE-NEXT: movdqa 160(%rdi), %xmm11 -; SSE-NEXT: movdqa 176(%rdi), %xmm3 -; SSE-NEXT: movdqa 64(%rdi), %xmm13 -; SSE-NEXT: movdqa 80(%rdi), %xmm7 +; SSE-NEXT: movdqa 128(%rdi), %xmm4 +; SSE-NEXT: movdqa 144(%rdi), %xmm7 +; SSE-NEXT: movdqa 160(%rdi), %xmm10 +; SSE-NEXT: movdqa 176(%rdi), %xmm12 +; SSE-NEXT: movdqa 64(%rdi), %xmm6 +; SSE-NEXT: movdqa 80(%rdi), %xmm13 ; SSE-NEXT: movdqa 96(%rdi), %xmm2 ; SSE-NEXT: movdqa 112(%rdi), %xmm1 -; SSE-NEXT: movdqa {{.*#+}} xmm5 = [255,0,255,0,255,0,255,0] +; SSE-NEXT: movdqa {{.*#+}} xmm9 = [255,0,255,0,255,0,255,0] ; SSE-NEXT: movdqa %xmm1, %xmm0 -; SSE-NEXT: movdqa %xmm1, %xmm4 -; SSE-NEXT: pand %xmm5, %xmm0 +; SSE-NEXT: movdqa %xmm1, %xmm3 +; SSE-NEXT: pand %xmm9, %xmm0 ; SSE-NEXT: movdqa %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm2, %xmm9 -; SSE-NEXT: pand %xmm5, %xmm1 +; SSE-NEXT: movdqa %xmm2, %xmm5 +; SSE-NEXT: pand %xmm9, %xmm1 ; SSE-NEXT: packuswb %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: pand %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm13, %xmm2 -; SSE-NEXT: pand %xmm5, %xmm2 +; SSE-NEXT: movdqa %xmm13, %xmm0 +; SSE-NEXT: pand %xmm9, %xmm0 +; SSE-NEXT: movdqa %xmm6, %xmm2 +; SSE-NEXT: pand %xmm9, %xmm2 ; SSE-NEXT: packuswb %xmm0, %xmm2 ; SSE-NEXT: packuswb %xmm1, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm3, %xmm0 -; SSE-NEXT: pand %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm11, %xmm1 -; SSE-NEXT: pand %xmm5, %xmm1 +; SSE-NEXT: movdqa %xmm12, %xmm0 +; SSE-NEXT: pand %xmm9, %xmm0 +; SSE-NEXT: movdqa %xmm10, %xmm1 +; SSE-NEXT: pand %xmm9, %xmm1 ; SSE-NEXT: packuswb %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: pand %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm15, %xmm2 -; SSE-NEXT: pand %xmm5, %xmm2 +; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: pand %xmm9, %xmm0 +; SSE-NEXT: movdqa %xmm4, %xmm2 +; SSE-NEXT: pand %xmm9, %xmm2 ; SSE-NEXT: packuswb %xmm0, %xmm2 ; SSE-NEXT: packuswb %xmm1, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: pand %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm6, %xmm1 -; SSE-NEXT: pand %xmm5, %xmm1 +; SSE-NEXT: pand %xmm9, %xmm0 +; SSE-NEXT: movdqa %xmm15, %xmm1 +; SSE-NEXT: pand %xmm9, %xmm1 ; SSE-NEXT: packuswb %xmm0, %xmm1 ; SSE-NEXT: movdqa %xmm8, %xmm0 -; SSE-NEXT: pand %xmm5, %xmm0 -; SSE-NEXT: movdqa (%rdi), %xmm12 -; SSE-NEXT: movdqa %xmm12, %xmm2 -; SSE-NEXT: pand %xmm5, %xmm2 +; SSE-NEXT: pand %xmm9, %xmm0 +; SSE-NEXT: movdqa (%rdi), %xmm11 +; SSE-NEXT: movdqa %xmm11, %xmm2 +; SSE-NEXT: pand %xmm9, %xmm2 ; SSE-NEXT: packuswb %xmm0, %xmm2 ; SSE-NEXT: packuswb %xmm1, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 240(%rdi), %xmm8 -; SSE-NEXT: movdqa %xmm8, %xmm0 -; SSE-NEXT: pand %xmm5, %xmm0 +; SSE-NEXT: movdqa 240(%rdi), %xmm0 +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm9, %xmm0 ; SSE-NEXT: movdqa 224(%rdi), %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm5, %xmm1 +; SSE-NEXT: pand %xmm9, %xmm1 ; SSE-NEXT: packuswb %xmm0, %xmm1 ; SSE-NEXT: movdqa 208(%rdi), %xmm0 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm5, %xmm0 +; SSE-NEXT: pand %xmm9, %xmm0 ; SSE-NEXT: movdqa 192(%rdi), %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm2, %xmm5 -; SSE-NEXT: packuswb %xmm0, %xmm5 -; SSE-NEXT: packuswb %xmm1, %xmm5 -; SSE-NEXT: pxor %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm4, %xmm1 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm4, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: pand %xmm2, %xmm9 +; SSE-NEXT: packuswb %xmm0, %xmm9 +; SSE-NEXT: packuswb %xmm1, %xmm9 +; SSE-NEXT: movdqa %xmm3, %xmm1 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm3, %xmm0 +; SSE-NEXT: pxor %xmm8, %xmm8 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE-NEXT: movdqa %xmm9, %xmm1 -; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm9, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: movdqa %xmm5, %xmm1 +; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm5, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1] -; SSE-NEXT: packuswb %xmm4, %xmm9 -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1] +; SSE-NEXT: packuswb %xmm3, %xmm5 +; SSE-NEXT: movdqa %xmm13, %xmm1 +; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm13, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE-NEXT: movdqa %xmm13, %xmm1 -; SSE-NEXT: movdqa %xmm13, (%rsp) # 16-byte Spill -; SSE-NEXT: movdqa %xmm13, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: movdqa %xmm6, %xmm1 +; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm6, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm13[1,3,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm6[1,3,2,3,4,5,6,7] ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSE-NEXT: packuswb %xmm4, %xmm1 -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm9[0,3] +; SSE-NEXT: packuswb %xmm3, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm5[0,3] ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm3, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: movdqa %xmm12, %xmm1 +; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm12, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm11, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] +; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: movdqa %xmm10, %xmm1 +; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1],xmm11[2],xmm2[2],xmm11[3],xmm2[3],xmm11[4],xmm2[4],xmm11[5],xmm2[5],xmm11[6],xmm2[6],xmm11[7],xmm2[7] -; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm11[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1] -; SSE-NEXT: packuswb %xmm4, %xmm9 -; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] +; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm5[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1] +; SSE-NEXT: packuswb %xmm3, %xmm6 +; SSE-NEXT: movdqa %xmm7, (%rsp) # 16-byte Spill +; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm2[0],xmm10[1],xmm2[1],xmm10[2],xmm2[2],xmm10[3],xmm2[3],xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7] -; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm10[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE-NEXT: movdqa %xmm15, %xmm1 -; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm15, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: movdqa %xmm7, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] +; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm4, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE-NEXT: movdqa %xmm4, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm13[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSE-NEXT: packuswb %xmm4, %xmm1 -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm9[0,3] -; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm5[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1] +; SSE-NEXT: packuswb %xmm3, %xmm10 +; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[0,3],xmm6[0,3] +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm14, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE-NEXT: movdqa %xmm6, %xmm7 -; SSE-NEXT: movdqa %xmm6, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm8[0],xmm14[1],xmm8[1],xmm14[2],xmm8[2],xmm14[3],xmm8[3],xmm14[4],xmm8[4],xmm14[5],xmm8[5],xmm14[6],xmm8[6],xmm14[7],xmm8[7] +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm14[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: movdqa %xmm15, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm6, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm13[0],xmm0[1],xmm13[1] -; SSE-NEXT: packuswb %xmm4, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload -; SSE-NEXT: movdqa %xmm6, %xmm1 -; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm6, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm9[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1] -; SSE-NEXT: movdqa %xmm12, %xmm3 -; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15] -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm4[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm12, %xmm4 -; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7] -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm4[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm3[0],xmm15[1],xmm3[1] -; SSE-NEXT: packuswb %xmm1, %xmm15 -; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[0,3],xmm0[0,3] -; SSE-NEXT: movdqa %xmm8, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm0[1,3,2,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm15, %xmm12 +; SSE-NEXT: movdqa %xmm15, %xmm14 +; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm8[0],xmm12[1],xmm8[1],xmm12[2],xmm8[2],xmm12[3],xmm8[3],xmm12[4],xmm8[4],xmm12[5],xmm8[5],xmm12[6],xmm8[6],xmm12[7],xmm8[7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm12[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm6[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1] +; SSE-NEXT: packuswb %xmm3, %xmm0 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload +; SSE-NEXT: movdqa %xmm15, %xmm13 +; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm8[8],xmm13[9],xmm8[9],xmm13[10],xmm8[10],xmm13[11],xmm8[11],xmm13[12],xmm8[12],xmm13[13],xmm8[13],xmm13[14],xmm8[14],xmm13[15],xmm8[15] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm13[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3],xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm15[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm6[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] +; SSE-NEXT: movdqa %xmm11, %xmm7 +; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm8[8],xmm7[9],xmm8[9],xmm7[10],xmm8[10],xmm7[11],xmm8[11],xmm7[12],xmm8[12],xmm7[13],xmm8[13],xmm7[14],xmm8[14],xmm7[15],xmm8[15] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm7[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm11, %xmm6 +; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1],xmm6[2],xmm8[2],xmm6[3],xmm8[3],xmm6[4],xmm8[4],xmm6[5],xmm8[5],xmm6[6],xmm8[6],xmm6[7],xmm8[7] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1] +; SSE-NEXT: packuswb %xmm1, %xmm5 +; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,3],xmm0[0,3] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm8, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; SSE-NEXT: movdqa %xmm1, %xmm3 +; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3],xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7] +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7] ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm10, %xmm3 -; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] +; SSE-NEXT: movdqa %xmm2, %xmm4 +; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1],xmm4[2],xmm8[2],xmm4[3],xmm8[3],xmm4[4],xmm8[4],xmm4[5],xmm8[5],xmm4[6],xmm8[6],xmm4[7],xmm8[7] +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7] ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] ; SSE-NEXT: packuswb %xmm1, %xmm3 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload -; SSE-NEXT: movdqa %xmm11, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm11, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3],xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm1[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; SSE-NEXT: movdqa %xmm9, %xmm1 -; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm9, %xmm4 -; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; SSE-NEXT: movdqa %xmm0, %xmm4 +; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm8[8],xmm4[9],xmm8[9],xmm4[10],xmm8[10],xmm4[11],xmm8[11],xmm4[12],xmm8[12],xmm4[13],xmm8[13],xmm4[14],xmm8[14],xmm4[15],xmm8[15] ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3],xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7] +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm1[1,3,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; SSE-NEXT: packuswb %xmm0, %xmm1 +; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1] +; SSE-NEXT: packuswb %xmm2, %xmm1 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm3[0,3] -; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm0, %xmm14 +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm14[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm0[0,1,2,3,7,5,6,7] +; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm0, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm0, %xmm7 -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm7[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm3 -; SSE-NEXT: pand %xmm0, %xmm6 -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm6[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm0, %xmm12 -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm12[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm4[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm12 -; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[0,3],xmm3[0,3] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload ; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm4 +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload +; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm11[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm3 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload ; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm2[0,1,2,3,7,6,5,4] -; SSE-NEXT: movdqa (%rsp), %xmm2 # 16-byte Reload +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm2[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm3[0,1,2,3,7,5,6,7] ; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm4, %xmm2 -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,3],xmm3[0,3] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm8, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,3],xmm4[0,3] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload +; SSE-NEXT: # xmm4 = mem[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm3 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: pand %xmm0, %xmm4 -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm4 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,0,3,2,4,5,6,7] ; SSE-NEXT: packuswb %xmm3, %xmm4 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload +; SSE-NEXT: # xmm8 = mem[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm3 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm3[0,1,2,3,7,6,5,4] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm8 +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm8[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm11 +; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[0,3],xmm4[0,3] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload +; SSE-NEXT: # xmm4 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] ; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm4 +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm4 +; SSE-NEXT: pshuflw $231, (%rsp), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload +; SSE-NEXT: # xmm8 = mem[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm6, %xmm3 -; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm4[0,3] +; SSE-NEXT: pand %xmm0, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,7,5,6,7] ; SSE-NEXT: pand %xmm0, %xmm8 -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm8[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm0, %xmm10 -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm10[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm6[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm4, %xmm7 -; SSE-NEXT: pand %xmm0, %xmm11 -; SSE-NEXT: pand %xmm0, %xmm9 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm11[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm9[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm8[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm14 +; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[0,3],xmm4[0,3] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload +; SSE-NEXT: # xmm4 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4] ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm4 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm4[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm0, %xmm6 -; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,3],xmm7[0,3] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm4 +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload +; SSE-NEXT: # xmm8 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm3 +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm8 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm0[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm8[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm0 +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,3],xmm4[0,3] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; SSE-NEXT: # xmm4 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload -; SSE-NEXT: # xmm7 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1] -; SSE-NEXT: packuswb %xmm4, %xmm7 -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: # xmm4 = mem[3,1,2,3] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm12[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm8 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm13[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm15[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm0[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1] -; SSE-NEXT: packuswb %xmm4, %xmm0 -; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,3],xmm7[0,3] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm7[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm6 +; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,3],xmm8[0,3] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; SSE-NEXT: # xmm4 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload ; SSE-NEXT: # xmm7 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: # xmm4 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload -; SSE-NEXT: # xmm8 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1] -; SSE-NEXT: packuswb %xmm7, %xmm8 +; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm7[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm8 +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; SSE-NEXT: # xmm4 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload -; SSE-NEXT: # xmm7 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm7[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: # xmm4 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload ; SSE-NEXT: # xmm7 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1] -; SSE-NEXT: packuswb %xmm10, %xmm7 +; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm7 ; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,3],xmm8[0,3] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; SSE-NEXT: # xmm4 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload ; SSE-NEXT: # xmm8 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: # xmm4 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload -; SSE-NEXT: # xmm10 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1] -; SSE-NEXT: packuswb %xmm8, %xmm10 +; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm8[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm3[0],xmm12[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm12 +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; SSE-NEXT: # xmm4 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload -; SSE-NEXT: # xmm8 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm8[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: # xmm4 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload ; SSE-NEXT: # xmm8 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1] -; SSE-NEXT: packuswb %xmm11, %xmm8 -; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[0,3],xmm10[0,3] +; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm8 +; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[0,3],xmm12[0,3] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; SSE-NEXT: # xmm4 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload -; SSE-NEXT: # xmm10 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: # xmm4 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload -; SSE-NEXT: # xmm11 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1] -; SSE-NEXT: packuswb %xmm10, %xmm11 +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload +; SSE-NEXT: # xmm12 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm12[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm3[0],xmm12[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm12 +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; SSE-NEXT: # xmm4 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload -; SSE-NEXT: # xmm10 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: # xmm4 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload ; SSE-NEXT: # xmm13 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm13[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm4[0],xmm13[1],xmm4[1] -; SSE-NEXT: packuswb %xmm10, %xmm13 -; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[0,3],xmm11[0,3] -; SSE-NEXT: movdqa %xmm5, 48(%rsi) -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: movaps %xmm4, (%rsi) -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: movaps %xmm4, 32(%rsi) -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: movaps %xmm4, 16(%rsi) +; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm3[0],xmm13[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm13 +; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[0,3],xmm12[0,3] +; SSE-NEXT: movdqa %xmm9, 48(%rsi) +; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; SSE-NEXT: movaps %xmm3, (%rsi) +; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; SSE-NEXT: movaps %xmm3, 32(%rsi) +; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; SSE-NEXT: movaps %xmm3, 16(%rsi) ; SSE-NEXT: movaps %xmm1, 48(%rdx) -; SSE-NEXT: movaps %xmm15, (%rdx) -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: movaps %xmm1, 32(%rdx) +; SSE-NEXT: movaps %xmm5, (%rdx) +; SSE-NEXT: movaps %xmm10, 32(%rdx) ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload ; SSE-NEXT: movaps %xmm1, 16(%rdx) -; SSE-NEXT: movaps %xmm6, 48(%rcx) -; SSE-NEXT: movaps %xmm3, 32(%rcx) -; SSE-NEXT: movaps %xmm2, 16(%rcx) -; SSE-NEXT: movaps %xmm12, (%rcx) +; SSE-NEXT: movaps %xmm0, 48(%rcx) +; SSE-NEXT: movaps %xmm14, 32(%rcx) +; SSE-NEXT: movaps %xmm11, 16(%rcx) +; SSE-NEXT: movaps %xmm2, (%rcx) ; SSE-NEXT: movaps %xmm13, 48(%r8) ; SSE-NEXT: movaps %xmm8, 32(%r8) ; SSE-NEXT: movaps %xmm7, 16(%r8) -; SSE-NEXT: movaps %xmm0, (%r8) -; SSE-NEXT: addq $664, %rsp # imm = 0x298 +; SSE-NEXT: movaps %xmm6, (%r8) +; SSE-NEXT: addq $600, %rsp # imm = 0x258 ; SSE-NEXT: retq ; ; AVX1-ONLY-LABEL: load_i8_stride4_vf64: diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll index a2ad944c66f42e5d6e54e17e22b7e02b6c7f518b..be34bbd254022275e646a263c3a40a0eda2d6dc2 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll @@ -104,9 +104,9 @@ define void @load_i8_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pand %xmm0, %xmm2 ; SSE-NEXT: pandn %xmm1, %xmm0 ; SSE-NEXT: por %xmm2, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm2, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [16711935,16711935,16711935,16711935] +; SSE-NEXT: pand %xmm2, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,1,4,5,6,7] @@ -130,10 +130,9 @@ define void @load_i8_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm9[2],xmm7[3],xmm9[3] ; SSE-NEXT: packuswb %xmm7, %xmm7 ; SSE-NEXT: por %xmm7, %xmm4 -; SSE-NEXT: movaps %xmm6, %xmm7 -; SSE-NEXT: andps %xmm2, %xmm7 -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm6[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,7,6,7] +; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm7 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[1,2,3,0,4,5,6,7] ; SSE-NEXT: packuswb %xmm7, %xmm7 @@ -145,8 +144,8 @@ define void @load_i8_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3] ; SSE-NEXT: packuswb %xmm6, %xmm6 ; SSE-NEXT: por %xmm1, %xmm8 -; SSE-NEXT: pand %xmm8, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,1,2,0] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[3,1,2,0] +; SSE-NEXT: pand %xmm2, %xmm1 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,0,3,4,5,6,7] @@ -225,10 +224,9 @@ define void @load_i8_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pand %xmm8, %xmm1 ; SSE-NEXT: pandn %xmm3, %xmm8 ; SSE-NEXT: por %xmm1, %xmm8 -; SSE-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: movdqa %xmm8, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[0,2,1,3] +; SSE-NEXT: movdqa {{.*#+}} xmm5 = [16711935,16711935,16711935,16711935] ; SSE-NEXT: pand %xmm5, %xmm1 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7] @@ -274,10 +272,9 @@ define void @load_i8_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: movdqa %xmm4, %xmm12 ; SSE-NEXT: pand %xmm11, %xmm12 ; SSE-NEXT: por %xmm9, %xmm12 -; SSE-NEXT: movdqa %xmm12, %xmm9 -; SSE-NEXT: pand %xmm5, %xmm9 -; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm12[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm5, %xmm9 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[1,2,3,0,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm13 = xmm9[0,1,2,3,5,5,5,5] @@ -318,8 +315,8 @@ define void @load_i8_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pand %xmm11, %xmm3 ; SSE-NEXT: pandn %xmm4, %xmm11 ; SSE-NEXT: por %xmm3, %xmm11 -; SSE-NEXT: pand %xmm11, %xmm5 -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm5[3,1,2,0] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm11[3,1,2,0] +; SSE-NEXT: pand %xmm5, %xmm3 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,1,0,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm3[2,1,0,3,4,5,6,7] @@ -470,79 +467,81 @@ define void @load_i8_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5) nounwind { ; SSE-LABEL: load_i8_stride6_vf16: ; SSE: # %bb.0: -; SSE-NEXT: movdqa 64(%rdi), %xmm11 -; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa 64(%rdi), %xmm10 +; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa (%rdi), %xmm5 -; SSE-NEXT: movdqa 16(%rdi), %xmm2 -; SSE-NEXT: movdqa 32(%rdi), %xmm8 +; SSE-NEXT: movdqa 16(%rdi), %xmm1 +; SSE-NEXT: movdqa 32(%rdi), %xmm7 ; SSE-NEXT: movdqa 48(%rdi), %xmm6 ; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,0,65535,65535,0,65535,65535] -; SSE-NEXT: movdqa %xmm4, %xmm7 -; SSE-NEXT: pandn %xmm8, %xmm7 -; SSE-NEXT: movdqa {{.*#+}} xmm10 = [65535,0,65535,65535,0,65535,65535,0] -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: pandn %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm4, %xmm0 -; SSE-NEXT: pandn %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pandn %xmm7, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,0,65535,65535,0,65535,65535,0] +; SSE-NEXT: movdqa %xmm2, %xmm3 +; SSE-NEXT: pandn %xmm6, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm4, %xmm3 +; SSE-NEXT: pandn %xmm6, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm4, %xmm6 -; SSE-NEXT: por %xmm7, %xmm6 -; SSE-NEXT: movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: por %xmm0, %xmm6 ; SSE-NEXT: movdqa %xmm6, %xmm0 -; SSE-NEXT: pand %xmm7, %xmm0 +; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,3,2,3] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,5] ; SSE-NEXT: packuswb %xmm3, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm9 = [65535,65535,65535,0,0,0,65535,65535] +; SSE-NEXT: movdqa {{.*#+}} xmm8 = [65535,65535,65535,0,0,0,65535,65535] +; SSE-NEXT: movdqa %xmm8, %xmm9 ; SSE-NEXT: pandn %xmm0, %xmm9 -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: pandn %xmm2, %xmm10 -; SSE-NEXT: pand %xmm4, %xmm11 -; SSE-NEXT: movdqa %xmm4, %xmm3 -; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: movdqa %xmm2, %xmm11 +; SSE-NEXT: pandn %xmm1, %xmm11 +; SSE-NEXT: pand %xmm4, %xmm10 +; SSE-NEXT: movdqa %xmm4, %xmm2 +; SSE-NEXT: pandn %xmm1, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm1, %xmm2 ; SSE-NEXT: movdqa %xmm5, %xmm14 ; SSE-NEXT: pand %xmm4, %xmm14 -; SSE-NEXT: movdqa 80(%rdi), %xmm1 -; SSE-NEXT: movdqa %xmm1, %xmm13 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa 80(%rdi), %xmm3 +; SSE-NEXT: movdqa %xmm3, %xmm13 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm4, %xmm13 -; SSE-NEXT: movdqa %xmm8, %xmm15 -; SSE-NEXT: pand %xmm4, %xmm8 +; SSE-NEXT: movdqa %xmm7, %xmm15 +; SSE-NEXT: pand %xmm4, %xmm7 ; SSE-NEXT: pand %xmm4, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm4, %xmm12 ; SSE-NEXT: movdqa %xmm4, %xmm2 ; SSE-NEXT: pandn %xmm5, %xmm4 ; SSE-NEXT: pand %xmm0, %xmm5 -; SSE-NEXT: por %xmm10, %xmm5 -; SSE-NEXT: movdqa %xmm5, %xmm10 -; SSE-NEXT: pand %xmm7, %xmm10 -; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,2,1,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,3,2,1,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm10[0,1,2,3,4,7,6,7] +; SSE-NEXT: por %xmm11, %xmm5 +; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm5[0,2,1,3] +; SSE-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm1, %xmm11 +; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,6,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[0,2,1,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[0,3,2,1,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm11[0,1,2,3,4,7,6,7] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE-NEXT: pand %xmm8, %xmm0 ; SSE-NEXT: por %xmm9, %xmm0 -; SSE-NEXT: pandn %xmm1, %xmm12 -; SSE-NEXT: por %xmm12, %xmm11 -; SSE-NEXT: movdqa %xmm11, %xmm9 -; SSE-NEXT: pand %xmm7, %xmm9 -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[3,1,2,0] +; SSE-NEXT: pandn %xmm3, %xmm12 +; SSE-NEXT: por %xmm12, %xmm10 +; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm10[3,1,2,0] +; SSE-NEXT: pand %xmm1, %xmm9 +; SSE-NEXT: movdqa %xmm1, %xmm3 ; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,3,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,7,6,5] ; SSE-NEXT: packuswb %xmm9, %xmm9 -; SSE-NEXT: movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] -; SSE-NEXT: movdqa %xmm10, %xmm1 -; SSE-NEXT: pandn %xmm9, %xmm1 -; SSE-NEXT: pand %xmm10, %xmm0 -; SSE-NEXT: por %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{.*#+}} xmm11 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] +; SSE-NEXT: movdqa %xmm11, %xmm12 +; SSE-NEXT: pandn %xmm9, %xmm12 +; SSE-NEXT: pand %xmm11, %xmm0 +; SSE-NEXT: por %xmm0, %xmm12 ; SSE-NEXT: pxor %xmm9, %xmm9 ; SSE-NEXT: movdqa %xmm6, %xmm0 ; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm9[8],xmm0[9],xmm9[9],xmm0[10],xmm9[10],xmm0[11],xmm9[11],xmm0[12],xmm9[12],xmm0[13],xmm9[13],xmm0[14],xmm9[14],xmm0[15],xmm9[15] @@ -568,15 +567,14 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pandn %xmm0, %xmm6 ; SSE-NEXT: por %xmm5, %xmm6 ; SSE-NEXT: packuswb %xmm6, %xmm6 -; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,65535,0,0,0,65535,65535] -; SSE-NEXT: pand %xmm0, %xmm6 -; SSE-NEXT: pandn %xmm1, %xmm0 -; SSE-NEXT: por %xmm0, %xmm6 -; SSE-NEXT: movdqa %xmm11, %xmm0 +; SSE-NEXT: pand %xmm8, %xmm6 +; SSE-NEXT: pandn %xmm1, %xmm8 +; SSE-NEXT: por %xmm8, %xmm6 +; SSE-NEXT: movdqa %xmm10, %xmm0 ; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3],xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5] -; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm9[8],xmm11[9],xmm9[9],xmm11[10],xmm9[10],xmm11[11],xmm9[11],xmm11[12],xmm9[12],xmm11[13],xmm9[13],xmm11[14],xmm9[14],xmm11[15],xmm9[15] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm11[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm10[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,7,6,4] ; SSE-NEXT: movdqa {{.*#+}} xmm5 = [65535,65535,65535,0,65535,65535,0,65535] @@ -584,14 +582,14 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pandn %xmm0, %xmm5 ; SSE-NEXT: por %xmm1, %xmm5 ; SSE-NEXT: packuswb %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm10, %xmm12 -; SSE-NEXT: pandn %xmm0, %xmm12 -; SSE-NEXT: pand %xmm10, %xmm6 -; SSE-NEXT: por %xmm6, %xmm12 +; SSE-NEXT: movdqa %xmm11, %xmm10 +; SSE-NEXT: pandn %xmm0, %xmm10 +; SSE-NEXT: pand %xmm11, %xmm6 +; SSE-NEXT: por %xmm6, %xmm10 ; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm15 ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload ; SSE-NEXT: movdqa %xmm15, %xmm0 -; SSE-NEXT: pand %xmm7, %xmm0 +; SSE-NEXT: pand %xmm3, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,7,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,3,4,5,6,7] @@ -599,11 +597,11 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,5,6] ; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: por %xmm3, %xmm14 -; SSE-NEXT: movdqa %xmm14, %xmm1 -; SSE-NEXT: pand %xmm7, %xmm1 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm14[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm3, %xmm1 +; SSE-NEXT: movdqa %xmm3, %xmm8 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] @@ -613,117 +611,118 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pandn %xmm1, %xmm5 ; SSE-NEXT: pand %xmm3, %xmm0 ; SSE-NEXT: por %xmm0, %xmm5 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload -; SSE-NEXT: pandn %xmm11, %xmm2 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload +; SSE-NEXT: pandn %xmm6, %xmm2 ; SSE-NEXT: por %xmm2, %xmm13 -; SSE-NEXT: movdqa %xmm13, %xmm0 -; SSE-NEXT: pand %xmm7, %xmm0 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm13[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] +; SSE-NEXT: pand %xmm8, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa %xmm10, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm10, %xmm5 -; SSE-NEXT: por %xmm5, %xmm1 +; SSE-NEXT: movdqa %xmm11, %xmm8 +; SSE-NEXT: pandn %xmm0, %xmm8 +; SSE-NEXT: pand %xmm11, %xmm5 +; SSE-NEXT: por %xmm5, %xmm8 ; SSE-NEXT: movdqa %xmm15, %xmm0 ; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3],xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7] ; SSE-NEXT: punpckhbw {{.*#+}} xmm15 = xmm15[8],xmm9[8],xmm15[9],xmm9[9],xmm15[10],xmm9[10],xmm15[11],xmm9[11],xmm15[12],xmm9[12],xmm15[13],xmm9[13],xmm15[14],xmm9[14],xmm15[15],xmm9[15] -; SSE-NEXT: movdqa %xmm15, %xmm2 -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[3,0] -; SSE-NEXT: movaps %xmm0, %xmm5 -; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm2[0,2] +; SSE-NEXT: movdqa %xmm15, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[3,0] +; SSE-NEXT: movaps %xmm0, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[0,2] ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm15[0,0] ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm15[2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm5[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm0, %xmm2 +; SSE-NEXT: packuswb %xmm0, %xmm1 ; SSE-NEXT: movdqa %xmm14, %xmm0 ; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3],xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,1,4,5,6,7] ; SSE-NEXT: punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm9[8],xmm14[9],xmm9[9],xmm14[10],xmm9[10],xmm14[11],xmm9[11],xmm14[12],xmm9[12],xmm14[13],xmm9[13],xmm14[14],xmm9[14],xmm14[15],xmm9[15] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm14[0,3,2,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,1,3,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,7,7,7] -; SSE-NEXT: movdqa {{.*#+}} xmm6 = [0,65535,65535,0,65535,65535,65535,65535] -; SSE-NEXT: pand %xmm6, %xmm5 -; SSE-NEXT: pandn %xmm0, %xmm6 -; SSE-NEXT: por %xmm5, %xmm6 -; SSE-NEXT: pand %xmm3, %xmm2 -; SSE-NEXT: packuswb %xmm6, %xmm6 -; SSE-NEXT: pandn %xmm6, %xmm3 -; SSE-NEXT: por %xmm2, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm14[0,3,2,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7] +; SSE-NEXT: movdqa {{.*#+}} xmm5 = [0,65535,65535,0,65535,65535,65535,65535] +; SSE-NEXT: pand %xmm5, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm5 +; SSE-NEXT: por %xmm2, %xmm5 +; SSE-NEXT: pand %xmm3, %xmm1 +; SSE-NEXT: packuswb %xmm5, %xmm5 +; SSE-NEXT: pandn %xmm5, %xmm3 +; SSE-NEXT: por %xmm1, %xmm3 ; SSE-NEXT: movdqa %xmm13, %xmm0 ; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm9[8],xmm0[9],xmm9[9],xmm0[10],xmm9[10],xmm0[11],xmm9[11],xmm0[12],xmm9[12],xmm0[13],xmm9[13],xmm0[14],xmm9[14],xmm0[15],xmm9[15] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,5] ; SSE-NEXT: punpcklbw {{.*#+}} xmm13 = xmm13[0],xmm9[0],xmm13[1],xmm9[1],xmm13[2],xmm9[2],xmm13[3],xmm9[3],xmm13[4],xmm9[4],xmm13[5],xmm9[5],xmm13[6],xmm9[6],xmm13[7],xmm9[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm13[0,2,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,7] -; SSE-NEXT: movdqa {{.*#+}} xmm5 = [65535,65535,65535,65535,0,65535,65535,0] -; SSE-NEXT: pand %xmm5, %xmm2 -; SSE-NEXT: pandn %xmm0, %xmm5 -; SSE-NEXT: por %xmm2, %xmm5 -; SSE-NEXT: pand %xmm10, %xmm3 -; SSE-NEXT: packuswb %xmm5, %xmm0 -; SSE-NEXT: pandn %xmm0, %xmm10 -; SSE-NEXT: por %xmm3, %xmm10 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm8, %xmm0 -; SSE-NEXT: pand %xmm7, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm13[0,2,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,7,7] +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535,0,65535,65535,0] +; SSE-NEXT: pand %xmm2, %xmm1 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: por %xmm1, %xmm2 +; SSE-NEXT: pand %xmm11, %xmm3 +; SSE-NEXT: packuswb %xmm2, %xmm0 +; SSE-NEXT: pandn %xmm0, %xmm11 +; SSE-NEXT: por %xmm3, %xmm11 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm2, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2] -; SSE-NEXT: packuswb %xmm2, %xmm0 +; SSE-NEXT: packuswb %xmm1, %xmm0 ; SSE-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255] -; SSE-NEXT: movdqa %xmm3, %xmm2 -; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm3, %xmm1 +; SSE-NEXT: pandn %xmm0, %xmm1 ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm4, %xmm0 -; SSE-NEXT: pand %xmm7, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm4[3,1,2,0] +; SSE-NEXT: pand %xmm2, %xmm0 +; SSE-NEXT: movdqa %xmm2, %xmm5 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm0[2,1,0,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm5, %xmm5 -; SSE-NEXT: pand %xmm3, %xmm5 -; SSE-NEXT: por %xmm2, %xmm5 +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm0[2,1,0,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: pand %xmm3, %xmm2 +; SSE-NEXT: por %xmm1, %xmm2 ; SSE-NEXT: movdqa {{.*#+}} xmm13 = [65535,0,65535,65535,0,65535,65535,0] -; SSE-NEXT: pand %xmm13, %xmm11 +; SSE-NEXT: movdqa %xmm6, %xmm1 +; SSE-NEXT: pand %xmm13, %xmm1 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,65535,65535,65535,0,0,0] -; SSE-NEXT: pand %xmm0, %xmm5 -; SSE-NEXT: por %xmm11, %xmm13 -; SSE-NEXT: pand %xmm13, %xmm7 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm7[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,1,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7] -; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: por %xmm1, %xmm13 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm13[0,2,1,3] +; SSE-NEXT: pand %xmm5, %xmm1 +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,1,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7] +; SSE-NEXT: packuswb %xmm1, %xmm1 ; SSE-NEXT: movdqa %xmm0, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: por %xmm5, %xmm6 -; SSE-NEXT: movdqa %xmm8, %xmm2 -; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm9[8],xmm2[9],xmm9[9],xmm2[10],xmm9[10],xmm2[11],xmm9[11],xmm2[12],xmm9[12],xmm2[13],xmm9[13],xmm2[14],xmm9[14],xmm2[15],xmm9[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3],xmm8[4],xmm9[4],xmm8[5],xmm9[5],xmm8[6],xmm9[6],xmm8[7],xmm9[7] -; SSE-NEXT: movdqa %xmm8, %xmm5 -; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[1,0],xmm2[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,0],xmm2[2,3] -; SSE-NEXT: psrlq $48, %xmm2 -; SSE-NEXT: psrldq {{.*#+}} xmm5 = xmm5[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm8[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm2[0,1,2,3,4,4,5,7] -; SSE-NEXT: packuswb %xmm5, %xmm7 +; SSE-NEXT: pandn %xmm1, %xmm6 +; SSE-NEXT: por %xmm2, %xmm6 +; SSE-NEXT: movdqa %xmm7, %xmm1 +; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm9[8],xmm1[9],xmm9[9],xmm1[10],xmm9[10],xmm1[11],xmm9[11],xmm1[12],xmm9[12],xmm1[13],xmm9[13],xmm1[14],xmm9[14],xmm1[15],xmm9[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3],xmm7[4],xmm9[4],xmm7[5],xmm9[5],xmm7[6],xmm9[6],xmm7[7],xmm9[7] +; SSE-NEXT: movdqa %xmm7, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[1,0],xmm1[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,0],xmm1[2,3] +; SSE-NEXT: psrlq $48, %xmm1 +; SSE-NEXT: psrldq {{.*#+}} xmm2 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm7[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,5,7] +; SSE-NEXT: packuswb %xmm2, %xmm1 ; SSE-NEXT: movdqa %xmm4, %xmm2 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm9[8],xmm2[9],xmm9[9],xmm2[10],xmm9[10],xmm2[11],xmm9[11],xmm2[12],xmm9[12],xmm2[13],xmm9[13],xmm2[14],xmm9[14],xmm2[15],xmm9[15] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3] @@ -738,29 +737,28 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: por %xmm4, %xmm2 ; SSE-NEXT: packuswb %xmm2, %xmm2 ; SSE-NEXT: pand %xmm3, %xmm2 -; SSE-NEXT: pandn %xmm7, %xmm3 +; SSE-NEXT: pandn %xmm1, %xmm3 ; SSE-NEXT: por %xmm3, %xmm2 -; SSE-NEXT: movdqa %xmm13, %xmm3 -; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3],xmm3[4],xmm9[4],xmm3[5],xmm9[5],xmm3[6],xmm9[6],xmm3[7],xmm9[7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,3,1,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm13, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3],xmm1[4],xmm9[4],xmm1[5],xmm9[5],xmm1[6],xmm9[6],xmm1[7],xmm9[7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,1,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7] ; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm9[8],xmm13[9],xmm9[9],xmm13[10],xmm9[10],xmm13[11],xmm9[11],xmm13[12],xmm9[12],xmm13[13],xmm9[13],xmm13[14],xmm9[14],xmm13[15],xmm9[15] -; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,65535,0,65535,0,0] -; SSE-NEXT: pand %xmm4, %xmm3 -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm13[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,7,4] -; SSE-NEXT: pandn %xmm5, %xmm4 -; SSE-NEXT: por %xmm3, %xmm4 +; SSE-NEXT: movdqa {{.*#+}} xmm3 = [65535,65535,65535,65535,0,65535,0,0] +; SSE-NEXT: pand %xmm3, %xmm1 +; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm13[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,7,4] +; SSE-NEXT: pandn %xmm4, %xmm3 +; SSE-NEXT: por %xmm1, %xmm3 ; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: packuswb %xmm4, %xmm3 -; SSE-NEXT: pandn %xmm3, %xmm0 +; SSE-NEXT: packuswb %xmm3, %xmm1 +; SSE-NEXT: pandn %xmm1, %xmm0 ; SSE-NEXT: por %xmm2, %xmm0 -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload -; SSE-NEXT: movaps %xmm2, (%rsi) -; SSE-NEXT: movdqa %xmm12, (%rdx) -; SSE-NEXT: movdqa %xmm1, (%rcx) -; SSE-NEXT: movdqa %xmm10, (%r8) +; SSE-NEXT: movdqa %xmm12, (%rsi) +; SSE-NEXT: movdqa %xmm10, (%rdx) +; SSE-NEXT: movdqa %xmm8, (%rcx) +; SSE-NEXT: movdqa %xmm11, (%r8) ; SSE-NEXT: movdqa %xmm6, (%r9) ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax ; SSE-NEXT: movdqa %xmm0, (%rax) @@ -1076,27 +1074,27 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5) nounwind { ; SSE-LABEL: load_i8_stride6_vf32: ; SSE: # %bb.0: -; SSE-NEXT: subq $280, %rsp # imm = 0x118 -; SSE-NEXT: movdqa 64(%rdi), %xmm10 -; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: subq $264, %rsp # imm = 0x108 +; SSE-NEXT: movdqa 64(%rdi), %xmm7 +; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa 80(%rdi), %xmm9 -; SSE-NEXT: movdqa (%rdi), %xmm14 -; SSE-NEXT: movdqa 16(%rdi), %xmm2 +; SSE-NEXT: movdqa (%rdi), %xmm12 +; SSE-NEXT: movdqa 16(%rdi), %xmm14 ; SSE-NEXT: movdqa 32(%rdi), %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa 48(%rdi), %xmm5 -; SSE-NEXT: movdqa {{.*#+}} xmm7 = [65535,65535,0,65535,65535,0,65535,65535] -; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm10 = [65535,65535,0,65535,65535,0,65535,65535] +; SSE-NEXT: movdqa %xmm10, %xmm0 ; SSE-NEXT: pandn %xmm1, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm12 = [65535,0,65535,65535,0,65535,65535,0] -; SSE-NEXT: movdqa %xmm12, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm11 = [65535,0,65535,65535,0,65535,65535,0] +; SSE-NEXT: movdqa %xmm11, %xmm1 ; SSE-NEXT: pandn %xmm5, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm1 +; SSE-NEXT: movdqa %xmm10, %xmm1 ; SSE-NEXT: pandn %xmm5, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm5, %xmm15 -; SSE-NEXT: pand %xmm7, %xmm15 +; SSE-NEXT: pand %xmm10, %xmm15 ; SSE-NEXT: por %xmm0, %xmm15 ; SSE-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255] ; SSE-NEXT: movdqa %xmm15, %xmm0 @@ -1107,238 +1105,237 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,5] ; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,0,0,0,65535,65535] -; SSE-NEXT: movdqa %xmm12, %xmm1 -; SSE-NEXT: pandn %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm14, %xmm11 -; SSE-NEXT: pand %xmm12, %xmm11 -; SSE-NEXT: por %xmm1, %xmm11 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,65535,65535,0,0,0,65535,65535] ; SSE-NEXT: movdqa %xmm11, %xmm1 +; SSE-NEXT: pandn %xmm14, %xmm1 +; SSE-NEXT: movdqa %xmm12, %xmm8 +; SSE-NEXT: pand %xmm11, %xmm8 +; SSE-NEXT: por %xmm1, %xmm8 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[0,2,1,3] ; SSE-NEXT: pand %xmm3, %xmm1 ; SSE-NEXT: movdqa %xmm3, %xmm6 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,7] ; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: pand %xmm4, %xmm1 -; SSE-NEXT: movdqa %xmm4, %xmm3 +; SSE-NEXT: pand %xmm2, %xmm1 +; SSE-NEXT: movdqa %xmm2, %xmm3 +; SSE-NEXT: movdqa %xmm2, %xmm5 ; SSE-NEXT: pandn %xmm0, %xmm3 ; SSE-NEXT: por %xmm3, %xmm1 -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: pandn %xmm9, %xmm0 -; SSE-NEXT: pand %xmm7, %xmm10 -; SSE-NEXT: por %xmm0, %xmm10 ; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: pandn %xmm9, %xmm0 +; SSE-NEXT: pand %xmm10, %xmm7 +; SSE-NEXT: por %xmm0, %xmm7 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm7[3,1,2,0] ; SSE-NEXT: pand %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm6, %xmm8 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] -; SSE-NEXT: movdqa %xmm5, %xmm3 -; SSE-NEXT: pandn %xmm0, %xmm3 -; SSE-NEXT: pand %xmm5, %xmm1 -; SSE-NEXT: por %xmm1, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 128(%rdi), %xmm13 -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: pandn %xmm13, %xmm0 -; SSE-NEXT: movdqa 144(%rdi), %xmm6 -; SSE-NEXT: movdqa %xmm12, %xmm3 -; SSE-NEXT: pandn %xmm6, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm3 -; SSE-NEXT: pandn %xmm6, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm7, %xmm6 -; SSE-NEXT: por %xmm0, %xmm6 -; SSE-NEXT: movdqa %xmm6, %xmm0 -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm0[0,3,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,3,2,3] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,5] -; SSE-NEXT: packuswb %xmm3, %xmm0 -; SSE-NEXT: movdqa %xmm4, %xmm8 -; SSE-NEXT: pandn %xmm0, %xmm8 -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: pandn %xmm14, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 112(%rdi), %xmm1 -; SSE-NEXT: movdqa %xmm12, %xmm4 -; SSE-NEXT: pandn %xmm1, %xmm4 -; SSE-NEXT: movdqa 160(%rdi), %xmm7 -; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm7 -; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: pandn %xmm2, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm14 -; SSE-NEXT: pandn %xmm9, %xmm12 -; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm9 -; SSE-NEXT: movdqa %xmm9, (%rsp) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: pandn %xmm1, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 96(%rdi), %xmm3 -; SSE-NEXT: movdqa %xmm3, %xmm5 -; SSE-NEXT: pand %xmm0, %xmm5 -; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 176(%rdi), %xmm5 -; SSE-NEXT: movdqa %xmm5, %xmm12 -; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm12 -; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; SSE-NEXT: movdqa %xmm9, %xmm12 -; SSE-NEXT: pand %xmm0, %xmm9 -; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm13, %xmm2 -; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] +; SSE-NEXT: movdqa %xmm3, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: pand %xmm3, %xmm1 +; SSE-NEXT: por %xmm1, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, %xmm2 -; SSE-NEXT: pand %xmm0, %xmm1 +; SSE-NEXT: movdqa 128(%rdi), %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pandn %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: pandn %xmm1, %xmm0 +; SSE-NEXT: movdqa 144(%rdi), %xmm1 +; SSE-NEXT: movdqa %xmm11, %xmm2 +; SSE-NEXT: pandn %xmm1, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm3, %xmm2 -; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 -; SSE-NEXT: por %xmm4, %xmm2 -; SSE-NEXT: movdqa %xmm2, %xmm4 -; SSE-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: movdqa %xmm10, %xmm2 +; SSE-NEXT: pandn %xmm1, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: pand %xmm10, %xmm2 +; SSE-NEXT: por %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: pand %xmm6, %xmm0 +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm0[0,3,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,3,2,3] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,5] +; SSE-NEXT: packuswb %xmm3, %xmm0 +; SSE-NEXT: movdqa %xmm5, %xmm6 +; SSE-NEXT: pandn %xmm0, %xmm6 +; SSE-NEXT: movdqa %xmm10, %xmm1 +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: pandn %xmm12, %xmm0 +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa 112(%rdi), %xmm0 +; SSE-NEXT: movdqa %xmm11, %xmm3 +; SSE-NEXT: pandn %xmm0, %xmm3 +; SSE-NEXT: movdqa 160(%rdi), %xmm5 +; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm10, %xmm5 +; SSE-NEXT: movdqa %xmm10, %xmm4 +; SSE-NEXT: pandn %xmm14, %xmm4 +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm10, %xmm12 +; SSE-NEXT: movdqa %xmm11, %xmm4 +; SSE-NEXT: pandn %xmm9, %xmm4 +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm9, %xmm11 +; SSE-NEXT: pand %xmm1, %xmm11 +; SSE-NEXT: movdqa %xmm1, %xmm4 +; SSE-NEXT: pandn %xmm0, %xmm4 +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa 96(%rdi), %xmm13 +; SSE-NEXT: movdqa %xmm13, %xmm4 ; SSE-NEXT: pand %xmm1, %xmm4 -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,1,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,6,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,3,2,1,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,7,6,7] -; SSE-NEXT: packuswb %xmm4, %xmm4 -; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 -; SSE-NEXT: por %xmm8, %xmm4 -; SSE-NEXT: pandn %xmm5, %xmm0 -; SSE-NEXT: por %xmm0, %xmm7 -; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa 176(%rdi), %xmm4 +; SSE-NEXT: movdqa %xmm4, %xmm10 +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm1, %xmm10 +; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload +; SSE-NEXT: movdqa %xmm9, %xmm10 +; SSE-NEXT: pand %xmm1, %xmm9 +; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm1, %xmm14 +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload +; SSE-NEXT: movdqa %xmm14, %xmm9 +; SSE-NEXT: pand %xmm1, %xmm14 +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm1, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5] -; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] -; SSE-NEXT: movdqa %xmm3, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm3, %xmm4 -; SSE-NEXT: por %xmm4, %xmm1 +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm1, %xmm14 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pxor %xmm9, %xmm9 -; SSE-NEXT: movdqa %xmm15, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm9[8],xmm0[9],xmm9[9],xmm0[10],xmm9[10],xmm0[11],xmm9[11],xmm0[12],xmm9[12],xmm0[13],xmm9[13],xmm0[14],xmm9[14],xmm0[15],xmm9[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3],xmm15[4],xmm9[4],xmm15[5],xmm9[5],xmm15[6],xmm9[6],xmm15[7],xmm9[7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm15[2,2,3,3] -; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3] -; SSE-NEXT: psrld $16, %xmm0 +; SSE-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill +; SSE-NEXT: pandn %xmm13, %xmm1 +; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm13, %xmm1 +; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; SSE-NEXT: por %xmm3, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,1,3] +; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm0, %xmm3 +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,1,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,3,2,1,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm3 +; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 +; SSE-NEXT: por %xmm6, %xmm3 +; SSE-NEXT: pandn %xmm4, %xmm14 +; SSE-NEXT: por %xmm14, %xmm5 +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm5[3,1,2,0] +; SSE-NEXT: pand %xmm0, %xmm4 +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,7,6,5] +; SSE-NEXT: packuswb %xmm4, %xmm4 +; SSE-NEXT: movdqa {{.*#+}} xmm13 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] +; SSE-NEXT: movdqa %xmm13, %xmm0 +; SSE-NEXT: pandn %xmm4, %xmm0 +; SSE-NEXT: pand %xmm13, %xmm3 +; SSE-NEXT: por %xmm3, %xmm0 +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pxor %xmm4, %xmm4 +; SSE-NEXT: movdqa %xmm15, %xmm3 +; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm4[0],xmm15[1],xmm4[1],xmm15[2],xmm4[2],xmm15[3],xmm4[3],xmm15[4],xmm4[4],xmm15[5],xmm4[5],xmm15[6],xmm4[6],xmm15[7],xmm4[7] +; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm15[2,2,3,3] +; SSE-NEXT: punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm3[0],xmm14[1],xmm3[1],xmm14[2],xmm3[2],xmm14[3],xmm3[3] +; SSE-NEXT: psrld $16, %xmm3 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm15[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,5,7,6,7] -; SSE-NEXT: punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm0[2],xmm15[3],xmm0[3] -; SSE-NEXT: packuswb %xmm15, %xmm5 -; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,0,0,0,65535,65535] -; SSE-NEXT: movdqa %xmm4, %xmm1 -; SSE-NEXT: pandn %xmm5, %xmm1 -; SSE-NEXT: movdqa %xmm11, %xmm5 -; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm9[8],xmm5[9],xmm9[9],xmm5[10],xmm9[10],xmm5[11],xmm9[11],xmm5[12],xmm9[12],xmm5[13],xmm9[13],xmm5[14],xmm9[14],xmm5[15],xmm9[15] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[2,1,0,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,1,1,1,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,7,6,7] +; SSE-NEXT: punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm3[2],xmm15[3],xmm3[3] +; SSE-NEXT: packuswb %xmm15, %xmm14 +; SSE-NEXT: movdqa {{.*#+}} xmm6 = [65535,65535,65535,0,0,0,65535,65535] +; SSE-NEXT: movdqa %xmm6, %xmm3 +; SSE-NEXT: pandn %xmm14, %xmm3 +; SSE-NEXT: movdqa %xmm8, %xmm14 +; SSE-NEXT: punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm4[8],xmm14[9],xmm4[9],xmm14[10],xmm4[10],xmm14[11],xmm4[11],xmm14[12],xmm4[12],xmm14[13],xmm4[13],xmm14[14],xmm4[14],xmm14[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm14[2,1,0,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm14[1,1,1,1,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,5,7,6,7] ; SSE-NEXT: movdqa {{.*#+}} xmm15 = [65535,65535,0,65535,0,0,65535,65535] ; SSE-NEXT: movdqa %xmm15, %xmm0 -; SSE-NEXT: pandn %xmm5, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3],xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm11[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,3,2,0,4,5,6,7] -; SSE-NEXT: pand %xmm15, %xmm5 -; SSE-NEXT: por %xmm0, %xmm5 -; SSE-NEXT: packuswb %xmm5, %xmm5 -; SSE-NEXT: pand %xmm4, %xmm5 -; SSE-NEXT: por %xmm1, %xmm5 -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3],xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7] +; SSE-NEXT: pandn %xmm14, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3],xmm8[4],xmm4[4],xmm8[5],xmm4[5],xmm8[6],xmm4[6],xmm8[7],xmm4[7] +; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm8[1,3,2,0,4,5,6,7] +; SSE-NEXT: pand %xmm15, %xmm14 +; SSE-NEXT: por %xmm0, %xmm14 +; SSE-NEXT: packuswb %xmm14, %xmm14 +; SSE-NEXT: pand %xmm6, %xmm14 +; SSE-NEXT: por %xmm3, %xmm14 +; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5] ; SSE-NEXT: movdqa {{.*#+}} xmm8 = [65535,65535,65535,0,65535,65535,0,65535] -; SSE-NEXT: movdqa %xmm8, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[3,1,2,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm8, %xmm3 +; SSE-NEXT: pandn %xmm0, %xmm3 +; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm4[8],xmm7[9],xmm4[9],xmm7[10],xmm4[10],xmm7[11],xmm4[11],xmm7[12],xmm4[12],xmm7[13],xmm4[13],xmm7[14],xmm4[14],xmm7[15],xmm4[15] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,7,6,4] ; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: por %xmm3, %xmm0 ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa %xmm3, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm3, %xmm5 -; SSE-NEXT: por %xmm5, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm6, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm9[8],xmm0[9],xmm9[9],xmm0[10],xmm9[10],xmm0[11],xmm9[11],xmm0[12],xmm9[12],xmm0[13],xmm9[13],xmm0[14],xmm9[14],xmm0[15],xmm9[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3],xmm6[4],xmm9[4],xmm6[5],xmm9[5],xmm6[6],xmm9[6],xmm6[7],xmm9[7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[2,2,3,3] -; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] -; SSE-NEXT: psrld $16, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm6[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,7,6,7] -; SSE-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm0[2],xmm5[3],xmm0[3] -; SSE-NEXT: packuswb %xmm5, %xmm1 +; SSE-NEXT: movdqa %xmm13, %xmm3 +; SSE-NEXT: pandn %xmm0, %xmm3 +; SSE-NEXT: pand %xmm13, %xmm14 +; SSE-NEXT: por %xmm14, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm2, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm9[8],xmm0[9],xmm9[9],xmm0[10],xmm9[10],xmm0[11],xmm9[11],xmm0[12],xmm9[12],xmm0[13],xmm9[13],xmm0[14],xmm9[14],xmm0[15],xmm9[15] +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,2,3,3] +; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3] +; SSE-NEXT: psrld $16, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,7,6,7] +; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSE-NEXT: packuswb %xmm2, %xmm3 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,7,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3],xmm2[4],xmm9[4],xmm2[5],xmm9[5],xmm2[6],xmm9[6],xmm2[7],xmm9[7] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,3,2,0,4,5,6,7] -; SSE-NEXT: pand %xmm15, %xmm2 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,3,2,0,4,5,6,7] +; SSE-NEXT: pand %xmm15, %xmm1 ; SSE-NEXT: pandn %xmm0, %xmm15 -; SSE-NEXT: por %xmm2, %xmm15 +; SSE-NEXT: por %xmm1, %xmm15 ; SSE-NEXT: packuswb %xmm15, %xmm15 -; SSE-NEXT: movdqa %xmm4, %xmm0 -; SSE-NEXT: pand %xmm4, %xmm15 -; SSE-NEXT: pandn %xmm1, %xmm0 -; SSE-NEXT: por %xmm0, %xmm15 -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3],xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7] +; SSE-NEXT: pand %xmm6, %xmm15 +; SSE-NEXT: pandn %xmm3, %xmm6 +; SSE-NEXT: por %xmm6, %xmm15 +; SSE-NEXT: movdqa %xmm5, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5] -; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm9[8],xmm7[9],xmm9[9],xmm7[10],xmm9[10],xmm7[11],xmm9[11],xmm7[12],xmm9[12],xmm7[13],xmm9[13],xmm7[14],xmm9[14],xmm7[15],xmm9[15] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm7[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm5[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,7,6,4] ; SSE-NEXT: pand %xmm8, %xmm1 ; SSE-NEXT: pandn %xmm0, %xmm8 ; SSE-NEXT: por %xmm1, %xmm8 ; SSE-NEXT: packuswb %xmm8, %xmm0 -; SSE-NEXT: movdqa %xmm3, %xmm1 +; SSE-NEXT: movdqa %xmm13, %xmm1 ; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm3, %xmm15 -; SSE-NEXT: movdqa %xmm3, %xmm8 +; SSE-NEXT: pand %xmm13, %xmm15 +; SSE-NEXT: movdqa %xmm13, %xmm7 ; SSE-NEXT: por %xmm15, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,0,65535,65535,0,65535,65535,0] -; SSE-NEXT: pand %xmm4, %xmm12 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm12, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm11 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm11, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm5 = [65535,0,65535,65535,0,65535,65535,0] +; SSE-NEXT: pand %xmm5, %xmm10 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm15 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm15, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,7,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,3,4,5,6,7] @@ -1346,11 +1343,10 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,4,5,6] ; SSE-NEXT: packuswb %xmm1, %xmm2 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7] +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm12[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm15, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,2,3,0,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,5,5,5] @@ -1360,27 +1356,26 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pandn %xmm1, %xmm3 ; SSE-NEXT: pand %xmm0, %xmm2 ; SSE-NEXT: por %xmm2, %xmm3 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload -; SSE-NEXT: movdqa (%rsp), %xmm15 # 16-byte Reload -; SSE-NEXT: por %xmm1, %xmm15 -; SSE-NEXT: movdqa %xmm15, %xmm1 -; SSE-NEXT: pand %xmm11, %xmm1 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7] +; SSE-NEXT: pandn %xmm14, %xmm1 +; SSE-NEXT: por %xmm1, %xmm11 +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm11[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7] +; SSE-NEXT: pand %xmm15, %xmm1 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,2,2,2,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4] ; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa %xmm8, %xmm2 +; SSE-NEXT: movdqa %xmm13, %xmm2 ; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: pand %xmm8, %xmm3 +; SSE-NEXT: pand %xmm13, %xmm3 ; SSE-NEXT: por %xmm3, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm4, %xmm13 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm13, %xmm1 -; SSE-NEXT: pand %xmm11, %xmm1 +; SSE-NEXT: pand %xmm5, %xmm9 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm9, %xmm1 +; SSE-NEXT: pand %xmm15, %xmm1 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,7,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,3,4,5,6,7] @@ -1388,12 +1383,11 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,5,6] ; SSE-NEXT: packuswb %xmm2, %xmm1 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm4, %xmm2 -; SSE-NEXT: pand %xmm11, %xmm2 -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm13[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm15, %xmm2 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,0,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5] @@ -1402,294 +1396,289 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pandn %xmm2, %xmm3 ; SSE-NEXT: pand %xmm0, %xmm1 ; SSE-NEXT: por %xmm1, %xmm3 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; SSE-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload -; SSE-NEXT: por %xmm1, %xmm10 -; SSE-NEXT: movdqa %xmm10, %xmm1 -; SSE-NEXT: pand %xmm11, %xmm1 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload +; SSE-NEXT: por %xmm1, %xmm8 +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm8[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7] +; SSE-NEXT: pand %xmm15, %xmm1 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,2,2,2,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4] ; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa %xmm8, %xmm2 +; SSE-NEXT: movdqa %xmm7, %xmm2 ; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: pand %xmm8, %xmm3 +; SSE-NEXT: pand %xmm7, %xmm3 ; SSE-NEXT: por %xmm3, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm12, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3],xmm1[4],xmm9[4],xmm1[5],xmm9[5],xmm1[6],xmm9[6],xmm1[7],xmm9[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm9[8],xmm12[9],xmm9[9],xmm12[10],xmm9[10],xmm12[11],xmm9[11],xmm12[12],xmm9[12],xmm12[13],xmm9[13],xmm12[14],xmm9[14],xmm12[15],xmm9[15] -; SSE-NEXT: movdqa %xmm12, %xmm3 -; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm1[3,0] -; SSE-NEXT: movaps %xmm1, %xmm5 -; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm3[0,2] -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm12[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm12[2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm5[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,2] +; SSE-NEXT: movdqa %xmm10, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm4[8],xmm10[9],xmm4[9],xmm10[10],xmm4[10],xmm10[11],xmm4[11],xmm10[12],xmm4[12],xmm10[13],xmm4[13],xmm10[14],xmm4[14],xmm10[15],xmm4[15] +; SSE-NEXT: movdqa %xmm10, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[3,0] +; SSE-NEXT: movaps %xmm1, %xmm3 +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[0,2] +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm10[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm10[2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm1, %xmm3 -; SSE-NEXT: movdqa %xmm14, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3],xmm1[4],xmm9[4],xmm1[5],xmm9[5],xmm1[6],xmm9[6],xmm1[7],xmm9[7] +; SSE-NEXT: packuswb %xmm1, %xmm2 +; SSE-NEXT: movdqa %xmm12, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm1[3,1,2,1,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm1[3,1,2,1,4,5,6,7] ; SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,65535,65535,0,65535,65535,65535,65535] -; SSE-NEXT: movdqa %xmm1, %xmm6 -; SSE-NEXT: pandn %xmm5, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm9[8],xmm14[9],xmm9[9],xmm14[10],xmm9[10],xmm14[11],xmm9[11],xmm14[12],xmm9[12],xmm14[13],xmm9[13],xmm14[14],xmm9[14],xmm14[15],xmm9[15] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm14[0,3,2,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,1,3,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,7,7,7] -; SSE-NEXT: pand %xmm1, %xmm5 -; SSE-NEXT: por %xmm6, %xmm5 -; SSE-NEXT: packuswb %xmm5, %xmm5 -; SSE-NEXT: movdqa %xmm0, %xmm6 -; SSE-NEXT: pandn %xmm5, %xmm6 -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: por %xmm3, %xmm6 -; SSE-NEXT: movdqa %xmm15, %xmm3 -; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm9[8],xmm3[9],xmm9[9],xmm3[10],xmm9[10],xmm3[11],xmm9[11],xmm3[12],xmm9[12],xmm3[13],xmm9[13],xmm3[14],xmm9[14],xmm3[15],xmm9[15] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,1] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm3[0,1,2,3,7,5,6,5] -; SSE-NEXT: movdqa {{.*#+}} xmm3 = [65535,65535,65535,65535,0,65535,65535,0] -; SSE-NEXT: movdqa %xmm3, %xmm7 -; SSE-NEXT: pandn %xmm5, %xmm7 -; SSE-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3],xmm15[4],xmm9[4],xmm15[5],xmm9[5],xmm15[6],xmm9[6],xmm15[7],xmm9[7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm15[0,2,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,7,7] -; SSE-NEXT: pand %xmm3, %xmm5 -; SSE-NEXT: por %xmm7, %xmm5 -; SSE-NEXT: packuswb %xmm5, %xmm5 -; SSE-NEXT: movdqa %xmm8, %xmm15 -; SSE-NEXT: pandn %xmm5, %xmm15 -; SSE-NEXT: pand %xmm8, %xmm6 -; SSE-NEXT: por %xmm6, %xmm15 -; SSE-NEXT: movdqa %xmm13, %xmm5 -; SSE-NEXT: pxor %xmm2, %xmm2 -; SSE-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm2[8],xmm13[9],xmm2[9],xmm13[10],xmm2[10],xmm13[11],xmm2[11],xmm13[12],xmm2[12],xmm13[13],xmm2[13],xmm13[14],xmm2[14],xmm13[15],xmm2[15] -; SSE-NEXT: movdqa %xmm13, %xmm6 -; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,0],xmm5[3,0] -; SSE-NEXT: movaps %xmm5, %xmm7 -; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,1],xmm6[0,2] -; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,0],xmm13[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,0],xmm13[2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm7[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,2] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,1,3,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm5, %xmm6 -; SSE-NEXT: movdqa %xmm4, %xmm5 -; SSE-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[3,1,2,1,4,5,6,7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15] -; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm4[0,3,2,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,3,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,7,7,7,7] -; SSE-NEXT: pand %xmm1, %xmm7 -; SSE-NEXT: pandn %xmm5, %xmm1 -; SSE-NEXT: por %xmm7, %xmm1 -; SSE-NEXT: pand %xmm0, %xmm6 +; SSE-NEXT: movdqa %xmm1, %xmm5 +; SSE-NEXT: pandn %xmm3, %xmm5 +; SSE-NEXT: punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm4[8],xmm12[9],xmm4[9],xmm12[10],xmm4[10],xmm12[11],xmm4[11],xmm12[12],xmm4[12],xmm12[13],xmm4[13],xmm12[14],xmm4[14],xmm12[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm12[0,3,2,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,7,7,7] +; SSE-NEXT: pand %xmm1, %xmm3 +; SSE-NEXT: por %xmm5, %xmm3 +; SSE-NEXT: packuswb %xmm3, %xmm3 +; SSE-NEXT: movdqa %xmm0, %xmm5 +; SSE-NEXT: pandn %xmm3, %xmm5 +; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: por %xmm2, %xmm5 +; SSE-NEXT: movdqa %xmm11, %xmm2 +; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm4[8],xmm2[9],xmm4[9],xmm2[10],xmm4[10],xmm2[11],xmm4[11],xmm2[12],xmm4[12],xmm2[13],xmm4[13],xmm2[14],xmm4[14],xmm2[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm2[0,1,2,3,7,5,6,5] +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535,0,65535,65535,0] +; SSE-NEXT: movdqa %xmm2, %xmm6 +; SSE-NEXT: pandn %xmm3, %xmm6 +; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3],xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm11[0,2,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,7,7] +; SSE-NEXT: pand %xmm2, %xmm3 +; SSE-NEXT: por %xmm6, %xmm3 +; SSE-NEXT: packuswb %xmm3, %xmm3 +; SSE-NEXT: movdqa %xmm7, %xmm6 +; SSE-NEXT: pandn %xmm3, %xmm6 +; SSE-NEXT: pand %xmm7, %xmm5 +; SSE-NEXT: por %xmm5, %xmm6 +; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm9, %xmm3 +; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15] +; SSE-NEXT: movdqa %xmm9, %xmm5 +; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,0],xmm3[3,0] +; SSE-NEXT: movaps %xmm3, %xmm6 +; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm5[0,2] +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm9[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm9[2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm6[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,0,2] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm5 +; SSE-NEXT: movdqa %xmm13, %xmm3 +; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,1,4,5,6,7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm4[8],xmm13[9],xmm4[9],xmm13[10],xmm4[10],xmm13[11],xmm4[11],xmm13[12],xmm4[12],xmm13[13],xmm4[13],xmm13[14],xmm4[14],xmm13[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm13[0,3,2,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,3,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,7,7,7] +; SSE-NEXT: pand %xmm1, %xmm6 +; SSE-NEXT: pandn %xmm3, %xmm1 +; SSE-NEXT: por %xmm6, %xmm1 +; SSE-NEXT: pand %xmm0, %xmm5 ; SSE-NEXT: packuswb %xmm1, %xmm1 ; SSE-NEXT: pandn %xmm1, %xmm0 -; SSE-NEXT: por %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm10, %xmm1 -; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] +; SSE-NEXT: por %xmm5, %xmm0 +; SSE-NEXT: movdqa %xmm8, %xmm1 +; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,5] -; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm2[0],xmm10[1],xmm2[1],xmm10[2],xmm2[2],xmm10[3],xmm2[3],xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm10[0,2,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,7,7] -; SSE-NEXT: pand %xmm3, %xmm5 -; SSE-NEXT: pandn %xmm1, %xmm3 -; SSE-NEXT: por %xmm5, %xmm3 -; SSE-NEXT: movdqa %xmm8, %xmm4 -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: packuswb %xmm3, %xmm1 -; SSE-NEXT: pandn %xmm1, %xmm4 -; SSE-NEXT: por %xmm0, %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm9, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3],xmm8[4],xmm4[4],xmm8[5],xmm4[5],xmm8[6],xmm4[6],xmm8[7],xmm4[7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm8[0,2,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,7,7] +; SSE-NEXT: pand %xmm2, %xmm3 +; SSE-NEXT: pandn %xmm1, %xmm2 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm7, %xmm13 +; SSE-NEXT: pand %xmm7, %xmm0 +; SSE-NEXT: packuswb %xmm2, %xmm1 +; SSE-NEXT: pandn %xmm1, %xmm13 +; SSE-NEXT: por %xmm0, %xmm13 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: pand %xmm15, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2] ; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255] -; SSE-NEXT: movdqa %xmm5, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255] +; SSE-NEXT: movdqa %xmm2, %xmm1 ; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm9[3,1,2,0] +; SSE-NEXT: pand %xmm15, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm0[2,1,0,3,4,5,6,7] ; SSE-NEXT: packuswb %xmm3, %xmm3 -; SSE-NEXT: pand %xmm5, %xmm3 +; SSE-NEXT: pand %xmm2, %xmm3 ; SSE-NEXT: por %xmm1, %xmm3 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload -; SSE-NEXT: movdqa {{.*#+}} xmm13 = [65535,0,65535,65535,0,65535,65535,0] -; SSE-NEXT: pand %xmm13, %xmm12 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm12, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] +; SSE-NEXT: movdqa %xmm14, %xmm11 +; SSE-NEXT: movdqa {{.*#+}} xmm12 = [65535,0,65535,65535,0,65535,65535,0] +; SSE-NEXT: pand %xmm12, %xmm11 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm11[0,2,1,3] +; SSE-NEXT: pand %xmm15, %xmm0 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7] -; SSE-NEXT: packuswb %xmm0, %xmm6 +; SSE-NEXT: packuswb %xmm0, %xmm5 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,65535,65535,65535,0,0,0] ; SSE-NEXT: movdqa %xmm0, %xmm8 -; SSE-NEXT: pandn %xmm6, %xmm8 +; SSE-NEXT: pandn %xmm5, %xmm8 ; SSE-NEXT: pand %xmm0, %xmm3 ; SSE-NEXT: por %xmm3, %xmm8 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload ; SSE-NEXT: movdqa %xmm14, %xmm3 -; SSE-NEXT: pand %xmm11, %xmm3 -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm3[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[2,1,2,3,4,5,6,7] +; SSE-NEXT: pand %xmm15, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm3[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,2] -; SSE-NEXT: packuswb %xmm6, %xmm3 -; SSE-NEXT: movdqa %xmm5, %xmm6 -; SSE-NEXT: pandn %xmm3, %xmm6 +; SSE-NEXT: packuswb %xmm5, %xmm3 +; SSE-NEXT: movdqa %xmm2, %xmm5 +; SSE-NEXT: pandn %xmm3, %xmm5 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm1, %xmm3 -; SSE-NEXT: pand %xmm11, %xmm3 -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,1,2,0] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[3,1,2,0] +; SSE-NEXT: pand %xmm15, %xmm3 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,1,0,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm3[2,1,0,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm7, %xmm7 -; SSE-NEXT: pand %xmm5, %xmm7 -; SSE-NEXT: por %xmm6, %xmm7 +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm3[2,1,0,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm6, %xmm6 +; SSE-NEXT: pand %xmm2, %xmm6 +; SSE-NEXT: por %xmm5, %xmm6 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; SSE-NEXT: pand %xmm13, %xmm3 -; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload -; SSE-NEXT: por %xmm3, %xmm13 -; SSE-NEXT: movdqa %xmm11, %xmm3 -; SSE-NEXT: pand %xmm13, %xmm3 -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,1,3] +; SSE-NEXT: pand %xmm12, %xmm3 +; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload +; SSE-NEXT: por %xmm3, %xmm12 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm12[0,2,1,3] +; SSE-NEXT: pand %xmm15, %xmm3 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,2,1,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,0,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7] -; SSE-NEXT: packuswb %xmm3, %xmm6 +; SSE-NEXT: packuswb %xmm3, %xmm5 ; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: pandn %xmm6, %xmm3 -; SSE-NEXT: pand %xmm0, %xmm7 -; SSE-NEXT: por %xmm7, %xmm3 -; SSE-NEXT: movdqa %xmm9, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3],xmm9[4],xmm2[4],xmm9[5],xmm2[5],xmm9[6],xmm2[6],xmm9[7],xmm2[7] -; SSE-NEXT: movdqa %xmm9, %xmm7 -; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[1,0],xmm6[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,0],xmm6[2,3] -; SSE-NEXT: psrlq $48, %xmm6 -; SSE-NEXT: psrldq {{.*#+}} xmm7 = xmm7[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; SSE-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm9[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,5,7] -; SSE-NEXT: packuswb %xmm7, %xmm6 -; SSE-NEXT: movdqa %xmm5, %xmm7 -; SSE-NEXT: pandn %xmm6, %xmm7 -; SSE-NEXT: movdqa %xmm10, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm6[0,1,2,3,5,5,5,5] -; SSE-NEXT: movdqa {{.*#+}} xmm6 = [65535,0,65535,65535,0,65535,65535,65535] -; SSE-NEXT: movdqa %xmm6, %xmm11 -; SSE-NEXT: pandn %xmm9, %xmm11 -; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm2[0],xmm10[1],xmm2[1],xmm10[2],xmm2[2],xmm10[3],xmm2[3],xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7] -; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm10[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm9[3,1,1,2,4,5,6,7] -; SSE-NEXT: pand %xmm6, %xmm10 -; SSE-NEXT: por %xmm11, %xmm10 -; SSE-NEXT: packuswb %xmm10, %xmm10 -; SSE-NEXT: pand %xmm5, %xmm10 -; SSE-NEXT: por %xmm7, %xmm10 -; SSE-NEXT: movdqa %xmm12, %xmm7 -; SSE-NEXT: punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm2[8],xmm12[9],xmm2[9],xmm12[10],xmm2[10],xmm12[11],xmm2[11],xmm12[12],xmm2[12],xmm12[13],xmm2[13],xmm12[14],xmm2[14],xmm12[15],xmm2[15] -; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm12[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm9[0,1,2,3,5,5,7,4] -; SSE-NEXT: movdqa {{.*#+}} xmm9 = [65535,65535,65535,65535,0,65535,0,0] -; SSE-NEXT: movdqa %xmm9, %xmm12 -; SSE-NEXT: pandn %xmm11, %xmm12 -; SSE-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3],xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7] -; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,3,1,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7] -; SSE-NEXT: pand %xmm9, %xmm7 -; SSE-NEXT: por %xmm7, %xmm12 -; SSE-NEXT: packuswb %xmm12, %xmm11 -; SSE-NEXT: movdqa %xmm0, %xmm7 -; SSE-NEXT: pandn %xmm11, %xmm7 -; SSE-NEXT: pand %xmm0, %xmm10 -; SSE-NEXT: por %xmm10, %xmm7 -; SSE-NEXT: movdqa %xmm14, %xmm10 -; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm2[8],xmm10[9],xmm2[9],xmm10[10],xmm2[10],xmm10[11],xmm2[11],xmm10[12],xmm2[12],xmm10[13],xmm2[13],xmm10[14],xmm2[14],xmm10[15],xmm2[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm2[0],xmm14[1],xmm2[1],xmm14[2],xmm2[2],xmm14[3],xmm2[3],xmm14[4],xmm2[4],xmm14[5],xmm2[5],xmm14[6],xmm2[6],xmm14[7],xmm2[7] -; SSE-NEXT: movdqa %xmm14, %xmm11 -; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[1,0],xmm10[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[2,0],xmm10[2,3] -; SSE-NEXT: psrlq $48, %xmm10 -; SSE-NEXT: psrldq {{.*#+}} xmm11 = xmm11[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; SSE-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm14[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,4,5,7] +; SSE-NEXT: pandn %xmm5, %xmm3 +; SSE-NEXT: pand %xmm0, %xmm6 +; SSE-NEXT: por %xmm6, %xmm3 +; SSE-NEXT: movdqa %xmm7, %xmm5 +; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3],xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7] +; SSE-NEXT: movdqa %xmm7, %xmm6 +; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[1,0],xmm5[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,0],xmm5[2,3] +; SSE-NEXT: psrlq $48, %xmm5 +; SSE-NEXT: psrldq {{.*#+}} xmm6 = xmm6[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3] +; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm7[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,4,5,7] +; SSE-NEXT: packuswb %xmm6, %xmm5 +; SSE-NEXT: movdqa %xmm2, %xmm6 +; SSE-NEXT: pandn %xmm5, %xmm6 +; SSE-NEXT: movdqa %xmm9, %xmm5 +; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm5[0,1,2,3,5,5,5,5] +; SSE-NEXT: movdqa {{.*#+}} xmm5 = [65535,0,65535,65535,0,65535,65535,65535] +; SSE-NEXT: movdqa %xmm5, %xmm10 +; SSE-NEXT: pandn %xmm7, %xmm10 +; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3],xmm9[4],xmm4[4],xmm9[5],xmm4[5],xmm9[6],xmm4[6],xmm9[7],xmm4[7] +; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm9[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm7[3,1,1,2,4,5,6,7] +; SSE-NEXT: pand %xmm5, %xmm9 +; SSE-NEXT: por %xmm10, %xmm9 +; SSE-NEXT: packuswb %xmm9, %xmm9 +; SSE-NEXT: pand %xmm2, %xmm9 +; SSE-NEXT: por %xmm6, %xmm9 +; SSE-NEXT: movdqa %xmm11, %xmm6 +; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm4[8],xmm11[9],xmm4[9],xmm11[10],xmm4[10],xmm11[11],xmm4[11],xmm11[12],xmm4[12],xmm11[13],xmm4[13],xmm11[14],xmm4[14],xmm11[15],xmm4[15] +; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm11[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm7[0,1,2,3,5,5,7,4] +; SSE-NEXT: movdqa {{.*#+}} xmm7 = [65535,65535,65535,65535,0,65535,0,0] +; SSE-NEXT: movdqa %xmm7, %xmm11 +; SSE-NEXT: pandn %xmm10, %xmm11 +; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,3,1,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7] +; SSE-NEXT: pand %xmm7, %xmm6 +; SSE-NEXT: por %xmm6, %xmm11 ; SSE-NEXT: packuswb %xmm11, %xmm10 -; SSE-NEXT: movdqa %xmm1, %xmm11 -; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm2[8],xmm11[9],xmm2[9],xmm11[10],xmm2[10],xmm11[11],xmm2[11],xmm11[12],xmm2[12],xmm11[13],xmm2[13],xmm11[14],xmm2[14],xmm11[15],xmm2[15] -; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[1,1,2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,5,5,5] -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; SSE-NEXT: pshufhw {{.*#+}} xmm12 = xmm1[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm12[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm12[3,1,1,2,4,5,6,7] -; SSE-NEXT: pand %xmm6, %xmm12 -; SSE-NEXT: pandn %xmm11, %xmm6 -; SSE-NEXT: por %xmm12, %xmm6 -; SSE-NEXT: packuswb %xmm6, %xmm6 -; SSE-NEXT: pand %xmm5, %xmm6 +; SSE-NEXT: movdqa %xmm0, %xmm6 +; SSE-NEXT: pandn %xmm10, %xmm6 +; SSE-NEXT: pand %xmm0, %xmm9 +; SSE-NEXT: por %xmm9, %xmm6 +; SSE-NEXT: movdqa %xmm14, %xmm11 +; SSE-NEXT: movdqa %xmm14, %xmm9 +; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3],xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7] +; SSE-NEXT: movdqa %xmm11, %xmm10 +; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[1,0],xmm9[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,0],xmm9[2,3] +; SSE-NEXT: psrlq $48, %xmm9 +; SSE-NEXT: psrldq {{.*#+}} xmm10 = xmm10[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm11[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,4,5,7] +; SSE-NEXT: packuswb %xmm10, %xmm9 +; SSE-NEXT: movdqa %xmm1, %xmm10 +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm4[8],xmm10[9],xmm4[9],xmm10[10],xmm4[10],xmm10[11],xmm4[11],xmm10[12],xmm4[12],xmm10[13],xmm4[13],xmm10[14],xmm4[14],xmm10[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5] +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm1[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[3,1,1,2,4,5,6,7] +; SSE-NEXT: pand %xmm5, %xmm11 ; SSE-NEXT: pandn %xmm10, %xmm5 -; SSE-NEXT: por %xmm5, %xmm6 -; SSE-NEXT: movdqa %xmm13, %xmm5 -; SSE-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm2[8],xmm13[9],xmm2[9],xmm13[10],xmm2[10],xmm13[11],xmm2[11],xmm13[12],xmm2[12],xmm13[13],xmm2[13],xmm13[14],xmm2[14],xmm13[15],xmm2[15] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm5[0,3,1,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: pand %xmm9, %xmm4 -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm13[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,7,4] -; SSE-NEXT: pandn %xmm5, %xmm9 -; SSE-NEXT: por %xmm4, %xmm9 -; SSE-NEXT: pand %xmm0, %xmm6 -; SSE-NEXT: packuswb %xmm9, %xmm4 -; SSE-NEXT: pandn %xmm4, %xmm0 -; SSE-NEXT: por %xmm6, %xmm0 -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: movaps %xmm4, 16(%rsi) -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: movaps %xmm4, (%rsi) +; SSE-NEXT: por %xmm11, %xmm5 +; SSE-NEXT: packuswb %xmm5, %xmm5 +; SSE-NEXT: pand %xmm2, %xmm5 +; SSE-NEXT: pandn %xmm9, %xmm2 +; SSE-NEXT: por %xmm2, %xmm5 +; SSE-NEXT: movdqa %xmm12, %xmm2 +; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm4[8],xmm12[9],xmm4[9],xmm12[10],xmm4[10],xmm12[11],xmm4[11],xmm12[12],xmm4[12],xmm12[13],xmm4[13],xmm12[14],xmm4[14],xmm12[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,1,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7] +; SSE-NEXT: pand %xmm7, %xmm2 +; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm12[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,7,4] +; SSE-NEXT: pandn %xmm4, %xmm7 +; SSE-NEXT: por %xmm2, %xmm7 +; SSE-NEXT: pand %xmm0, %xmm5 +; SSE-NEXT: packuswb %xmm7, %xmm2 +; SSE-NEXT: pandn %xmm2, %xmm0 +; SSE-NEXT: por %xmm5, %xmm0 +; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: movaps %xmm2, 16(%rsi) +; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: movaps %xmm2, (%rsi) ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload ; SSE-NEXT: movaps %xmm1, 16(%rdx) ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload @@ -1698,15 +1687,15 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: movaps %xmm1, 16(%rcx) ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload ; SSE-NEXT: movaps %xmm1, (%rcx) +; SSE-NEXT: movdqa %xmm13, 16(%r8) ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: movaps %xmm1, 16(%r8) -; SSE-NEXT: movdqa %xmm15, (%r8) +; SSE-NEXT: movaps %xmm1, (%r8) ; SSE-NEXT: movdqa %xmm3, 16(%r9) ; SSE-NEXT: movdqa %xmm8, (%r9) ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax ; SSE-NEXT: movdqa %xmm0, 16(%rax) -; SSE-NEXT: movdqa %xmm7, (%rax) -; SSE-NEXT: addq $280, %rsp # imm = 0x118 +; SSE-NEXT: movdqa %xmm6, (%rax) +; SSE-NEXT: addq $264, %rsp # imm = 0x108 ; SSE-NEXT: retq ; ; AVX1-ONLY-LABEL: load_i8_stride6_vf32: @@ -2299,7 +2288,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5) nounwind { ; SSE-LABEL: load_i8_stride6_vf64: ; SSE: # %bb.0: -; SSE-NEXT: subq $808, %rsp # imm = 0x328 +; SSE-NEXT: subq $792, %rsp # imm = 0x318 ; SSE-NEXT: movdqa 64(%rdi), %xmm4 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa 80(%rdi), %xmm5 @@ -2311,22 +2300,22 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: movdqa 32(%rdi), %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa 48(%rdi), %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm9 = [65535,65535,0,65535,65535,0,65535,65535] -; SSE-NEXT: movdqa %xmm9, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm13 = [65535,65535,0,65535,65535,0,65535,65535] +; SSE-NEXT: movdqa %xmm13, %xmm1 ; SSE-NEXT: pandn %xmm2, %xmm1 ; SSE-NEXT: movdqa {{.*#+}} xmm3 = [65535,0,65535,65535,0,65535,65535,0] ; SSE-NEXT: movdqa %xmm3, %xmm2 ; SSE-NEXT: pandn %xmm0, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm9, %xmm2 +; SSE-NEXT: movdqa %xmm13, %xmm2 ; SSE-NEXT: pandn %xmm0, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm9, %xmm0 +; SSE-NEXT: pand %xmm13, %xmm0 ; SSE-NEXT: por %xmm1, %xmm0 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255] ; SSE-NEXT: pand %xmm1, %xmm0 -; SSE-NEXT: movdqa %xmm1, %xmm11 +; SSE-NEXT: movdqa %xmm1, %xmm10 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] @@ -2339,9 +2328,8 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pand %xmm3, %xmm2 ; SSE-NEXT: por %xmm1, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm2, %xmm1 -; SSE-NEXT: pand %xmm11, %xmm1 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,1,3] +; SSE-NEXT: pand %xmm10, %xmm1 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7] @@ -2351,15 +2339,13 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: movdqa %xmm8, %xmm2 ; SSE-NEXT: pandn %xmm0, %xmm2 ; SSE-NEXT: por %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm9, %xmm6 -; SSE-NEXT: movdqa %xmm9, %xmm0 +; SSE-NEXT: movdqa %xmm13, %xmm0 ; SSE-NEXT: pandn %xmm5, %xmm0 -; SSE-NEXT: pand %xmm9, %xmm4 +; SSE-NEXT: pand %xmm13, %xmm4 ; SSE-NEXT: por %xmm0, %xmm4 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm4, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm4[3,1,2,0] +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5] @@ -2372,282 +2358,272 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa 320(%rdi), %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm9, %xmm0 +; SSE-NEXT: movdqa %xmm13, %xmm0 ; SSE-NEXT: pandn %xmm1, %xmm0 -; SSE-NEXT: movdqa 336(%rdi), %xmm13 +; SSE-NEXT: movdqa 336(%rdi), %xmm12 ; SSE-NEXT: movdqa %xmm3, %xmm1 -; SSE-NEXT: pandn %xmm13, %xmm1 +; SSE-NEXT: pandn %xmm12, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm9, %xmm1 -; SSE-NEXT: pandn %xmm13, %xmm1 +; SSE-NEXT: movdqa %xmm13, %xmm1 +; SSE-NEXT: pandn %xmm12, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm9, %xmm13 -; SSE-NEXT: por %xmm0, %xmm13 -; SSE-NEXT: movdqa %xmm13, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 +; SSE-NEXT: pand %xmm13, %xmm12 +; SSE-NEXT: por %xmm0, %xmm12 +; SSE-NEXT: movdqa %xmm12, %xmm0 +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,5] ; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: movdqa %xmm8, %xmm2 -; SSE-NEXT: pandn %xmm0, %xmm2 -; SSE-NEXT: movdqa 304(%rdi), %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm8, %xmm1 +; SSE-NEXT: pandn %xmm0, %xmm1 +; SSE-NEXT: movdqa 304(%rdi), %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm3, %xmm7 ; SSE-NEXT: movdqa %xmm3, %xmm0 -; SSE-NEXT: pandn %xmm1, %xmm0 -; SSE-NEXT: movdqa 288(%rdi), %xmm7 -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: pand %xmm3, %xmm1 -; SSE-NEXT: por %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm1, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] +; SSE-NEXT: pandn %xmm2, %xmm0 +; SSE-NEXT: movdqa 288(%rdi), %xmm6 +; SSE-NEXT: movdqa %xmm6, %xmm2 +; SSE-NEXT: pand %xmm3, %xmm2 +; SSE-NEXT: por %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,2,1,3] +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] ; SSE-NEXT: packuswb %xmm0, %xmm0 ; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: por %xmm2, %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 ; SSE-NEXT: movdqa 368(%rdi), %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm9, %xmm2 +; SSE-NEXT: movdqa %xmm13, %xmm2 ; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: movdqa 352(%rdi), %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm9, %xmm1 -; SSE-NEXT: por %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm1, %xmm2 -; SSE-NEXT: pand %xmm11, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[3,1,2,0] +; SSE-NEXT: movdqa 352(%rdi), %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm13, %xmm3 +; SSE-NEXT: por %xmm2, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[3,1,2,0] +; SSE-NEXT: pand %xmm10, %xmm2 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5] ; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm4, %xmm1 -; SSE-NEXT: pandn %xmm2, %xmm1 +; SSE-NEXT: movdqa %xmm4, %xmm3 +; SSE-NEXT: pandn %xmm2, %xmm3 ; SSE-NEXT: pand %xmm4, %xmm0 ; SSE-NEXT: movdqa %xmm4, %xmm9 -; SSE-NEXT: por %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: por %xmm0, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa 224(%rdi), %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm6, %xmm0 +; SSE-NEXT: movdqa %xmm13, %xmm0 ; SSE-NEXT: pandn %xmm1, %xmm0 -; SSE-NEXT: movdqa 240(%rdi), %xmm14 -; SSE-NEXT: movdqa %xmm3, %xmm1 -; SSE-NEXT: pandn %xmm14, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm6, %xmm1 -; SSE-NEXT: pandn %xmm14, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm6, %xmm14 -; SSE-NEXT: por %xmm0, %xmm14 -; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 +; SSE-NEXT: movdqa 240(%rdi), %xmm11 +; SSE-NEXT: movdqa %xmm7, %xmm2 +; SSE-NEXT: pandn %xmm11, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm13, %xmm2 +; SSE-NEXT: pandn %xmm11, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm13, %xmm11 +; SSE-NEXT: por %xmm0, %xmm11 +; SSE-NEXT: movdqa %xmm11, %xmm0 +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,5] ; SSE-NEXT: packuswb %xmm2, %xmm0 -; SSE-NEXT: movdqa %xmm8, %xmm5 ; SSE-NEXT: movdqa %xmm8, %xmm2 ; SSE-NEXT: pandn %xmm0, %xmm2 -; SSE-NEXT: movdqa 208(%rdi), %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm3, %xmm1 -; SSE-NEXT: movdqa %xmm3, %xmm0 -; SSE-NEXT: pandn %xmm4, %xmm0 +; SSE-NEXT: movdqa 208(%rdi), %xmm1 +; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: pandn %xmm1, %xmm0 ; SSE-NEXT: movdqa 192(%rdi), %xmm3 -; SSE-NEXT: movdqa %xmm3, %xmm4 -; SSE-NEXT: pand %xmm1, %xmm4 -; SSE-NEXT: movdqa %xmm1, %xmm8 -; SSE-NEXT: por %xmm0, %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm4, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] +; SSE-NEXT: movdqa %xmm3, %xmm1 +; SSE-NEXT: pand %xmm7, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,1,3] +; SSE-NEXT: movdqa %xmm10, %xmm1 +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pand %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm5, %xmm1 +; SSE-NEXT: pand %xmm8, %xmm0 +; SSE-NEXT: movdqa %xmm8, %xmm10 ; SSE-NEXT: por %xmm2, %xmm0 -; SSE-NEXT: movdqa 272(%rdi), %xmm15 -; SSE-NEXT: movdqa %xmm6, %xmm2 -; SSE-NEXT: pandn %xmm15, %xmm2 -; SSE-NEXT: movdqa 256(%rdi), %xmm12 -; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm6, %xmm12 -; SSE-NEXT: por %xmm2, %xmm12 -; SSE-NEXT: movdqa %xmm12, %xmm2 -; SSE-NEXT: pand %xmm11, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[3,1,2,0] +; SSE-NEXT: movdqa 272(%rdi), %xmm14 +; SSE-NEXT: movdqa %xmm13, %xmm2 +; SSE-NEXT: pandn %xmm14, %xmm2 +; SSE-NEXT: movdqa 256(%rdi), %xmm15 +; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm13, %xmm15 +; SSE-NEXT: por %xmm2, %xmm15 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm15[3,1,2,0] +; SSE-NEXT: pand %xmm1, %xmm2 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5] ; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm9, %xmm4 ; SSE-NEXT: pandn %xmm2, %xmm4 ; SSE-NEXT: pand %xmm9, %xmm0 ; SSE-NEXT: por %xmm0, %xmm4 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa 128(%rdi), %xmm2 ; SSE-NEXT: movdqa %xmm2, (%rsp) # 16-byte Spill -; SSE-NEXT: movdqa %xmm6, %xmm0 +; SSE-NEXT: movdqa %xmm13, %xmm0 ; SSE-NEXT: pandn %xmm2, %xmm0 -; SSE-NEXT: movdqa 144(%rdi), %xmm10 -; SSE-NEXT: movdqa %xmm8, %xmm4 -; SSE-NEXT: pandn %xmm10, %xmm4 +; SSE-NEXT: movdqa 144(%rdi), %xmm9 +; SSE-NEXT: movdqa %xmm7, %xmm4 +; SSE-NEXT: pandn %xmm9, %xmm4 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm6, %xmm4 -; SSE-NEXT: pandn %xmm10, %xmm4 +; SSE-NEXT: movdqa %xmm13, %xmm4 +; SSE-NEXT: pandn %xmm9, %xmm4 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm6, %xmm10 -; SSE-NEXT: por %xmm0, %xmm10 -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 +; SSE-NEXT: pand %xmm13, %xmm9 +; SSE-NEXT: por %xmm0, %xmm9 +; SSE-NEXT: movdqa %xmm9, %xmm0 +; SSE-NEXT: pand %xmm1, %xmm0 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,5] ; SSE-NEXT: packuswb %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm1, %xmm11 -; SSE-NEXT: pandn %xmm0, %xmm11 -; SSE-NEXT: movdqa %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm6, %xmm4 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: pandn %xmm1, %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm6, %xmm2 -; SSE-NEXT: pandn %xmm7, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm10 +; SSE-NEXT: movdqa %xmm13, %xmm0 +; SSE-NEXT: movdqa %xmm13, %xmm2 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload +; SSE-NEXT: pandn %xmm13, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm4 -; SSE-NEXT: movdqa %xmm6, %xmm2 +; SSE-NEXT: movdqa %xmm13, %xmm1 +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: pandn %xmm6, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm6, %xmm5 +; SSE-NEXT: movdqa %xmm0, %xmm2 ; SSE-NEXT: pandn %xmm3, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm3, %xmm5 +; SSE-NEXT: movdqa %xmm3, %xmm4 ; SSE-NEXT: movdqa 112(%rdi), %xmm6 -; SSE-NEXT: movdqa %xmm8, %xmm7 -; SSE-NEXT: movdqa %xmm8, %xmm9 -; SSE-NEXT: pandn %xmm6, %xmm9 -; SSE-NEXT: movdqa 160(%rdi), %xmm8 -; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm8 +; SSE-NEXT: movdqa %xmm7, %xmm2 +; SSE-NEXT: movdqa %xmm7, %xmm8 +; SSE-NEXT: pandn %xmm6, %xmm8 +; SSE-NEXT: movdqa 160(%rdi), %xmm7 +; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm7 ; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload -; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload +; SSE-NEXT: pandn %xmm13, %xmm3 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm0, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm1 +; SSE-NEXT: movdqa %xmm2, %xmm3 +; SSE-NEXT: movdqa %xmm2, %xmm1 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm0, %xmm2 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; SSE-NEXT: pandn %xmm3, %xmm1 +; SSE-NEXT: pandn %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm5 +; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm1, %xmm5 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; SSE-NEXT: pandn %xmm1, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, %xmm3 +; SSE-NEXT: movdqa %xmm0, %xmm2 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: pandn %xmm1, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pandn %xmm1, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm0, %xmm4 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm3 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: pandn %xmm4, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm4, %xmm3 -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: pandn %xmm4, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm5 +; SSE-NEXT: pandn %xmm14, %xmm5 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm3 -; SSE-NEXT: pandn %xmm15, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm15 -; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: pandn %xmm6, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm6, %xmm15 -; SSE-NEXT: movdqa 96(%rdi), %xmm5 -; SSE-NEXT: movdqa %xmm5, %xmm3 -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 176(%rdi), %xmm6 -; SSE-NEXT: movdqa %xmm6, %xmm3 -; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm14 +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: pandn %xmm6, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa 96(%rdi), %xmm4 +; SSE-NEXT: movdqa %xmm4, %xmm2 +; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa 176(%rdi), %xmm14 +; SSE-NEXT: movdqa %xmm14, %xmm2 +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm0, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm0, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm13 +; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm0, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm0, %xmm1 ; SSE-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill ; SSE-NEXT: movdqa %xmm0, %xmm1 -; SSE-NEXT: pand %xmm0, %xmm15 -; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm6 +; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm0, %xmm13 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pandn %xmm5, %xmm1 +; SSE-NEXT: pandn %xmm4, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm5, %xmm3 -; SSE-NEXT: pand %xmm7, %xmm3 -; SSE-NEXT: por %xmm9, %xmm3 -; SSE-NEXT: movdqa %xmm3, %xmm5 -; SSE-NEXT: movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm7, %xmm5 -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,1,3] +; SSE-NEXT: movdqa %xmm4, %xmm3 +; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 +; SSE-NEXT: por %xmm8, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,2,1,3] +; SSE-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm1, %xmm5 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,3,2,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,7] ; SSE-NEXT: packuswb %xmm5, %xmm5 -; SSE-NEXT: movdqa {{.*#+}} xmm9 = [65535,65535,65535,0,0,0,65535,65535] -; SSE-NEXT: pand %xmm9, %xmm5 -; SSE-NEXT: por %xmm11, %xmm5 -; SSE-NEXT: pandn %xmm6, %xmm0 -; SSE-NEXT: por %xmm0, %xmm8 -; SSE-NEXT: movdqa %xmm8, %xmm0 -; SSE-NEXT: pand %xmm7, %xmm0 -; SSE-NEXT: movdqa %xmm7, %xmm11 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] +; SSE-NEXT: movdqa {{.*#+}} xmm8 = [65535,65535,65535,0,0,0,65535,65535] +; SSE-NEXT: pand %xmm8, %xmm5 +; SSE-NEXT: por %xmm10, %xmm5 +; SSE-NEXT: pandn %xmm14, %xmm0 +; SSE-NEXT: por %xmm0, %xmm7 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm7[3,1,2,0] +; SSE-NEXT: pand %xmm1, %xmm0 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] -; SSE-NEXT: movdqa %xmm7, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] +; SSE-NEXT: movdqa %xmm10, %xmm1 ; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm7, %xmm5 +; SSE-NEXT: pand %xmm10, %xmm5 ; SSE-NEXT: por %xmm5, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pxor %xmm5, %xmm5 @@ -2658,11 +2634,11 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,2,3,3] ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3] ; SSE-NEXT: psrld $16, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm1[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,5,7,6,7] -; SSE-NEXT: punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm0[2],xmm15[3],xmm0[3] -; SSE-NEXT: packuswb %xmm15, %xmm4 -; SSE-NEXT: movdqa %xmm9, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm1[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,5,7,6,7] +; SSE-NEXT: punpckhdq {{.*#+}} xmm14 = xmm14[2],xmm0[2],xmm14[3],xmm0[3] +; SSE-NEXT: packuswb %xmm14, %xmm4 +; SSE-NEXT: movdqa %xmm8, %xmm1 ; SSE-NEXT: pandn %xmm4, %xmm1 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload ; SSE-NEXT: movdqa %xmm2, %xmm4 @@ -2671,48 +2647,48 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,1,1,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,7,6,7] ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,0,65535,0,0,65535,65535] -; SSE-NEXT: movdqa %xmm0, %xmm15 -; SSE-NEXT: pandn %xmm4, %xmm15 +; SSE-NEXT: movdqa %xmm0, %xmm14 +; SSE-NEXT: pandn %xmm4, %xmm14 ; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,3,2,0,4,5,6,7] ; SSE-NEXT: pand %xmm0, %xmm4 -; SSE-NEXT: por %xmm15, %xmm4 +; SSE-NEXT: por %xmm14, %xmm4 ; SSE-NEXT: packuswb %xmm4, %xmm4 -; SSE-NEXT: pand %xmm9, %xmm4 +; SSE-NEXT: pand %xmm8, %xmm4 ; SSE-NEXT: por %xmm1, %xmm4 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload ; SSE-NEXT: movdqa %xmm6, %xmm1 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7] -; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm1[0,1,2,3,5,5,5,5] +; SSE-NEXT: pshufhw {{.*#+}} xmm14 = xmm1[0,1,2,3,5,5,5,5] ; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,65535,65535,0,65535,65535,0,65535] ; SSE-NEXT: movdqa %xmm2, %xmm1 -; SSE-NEXT: pandn %xmm15, %xmm1 +; SSE-NEXT: pandn %xmm14, %xmm1 ; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15] -; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm6[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm15[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,5,7,6,4] -; SSE-NEXT: pand %xmm2, %xmm15 -; SSE-NEXT: por %xmm1, %xmm15 -; SSE-NEXT: packuswb %xmm15, %xmm1 -; SSE-NEXT: movdqa %xmm7, %xmm15 -; SSE-NEXT: pandn %xmm1, %xmm15 -; SSE-NEXT: pand %xmm7, %xmm4 -; SSE-NEXT: por %xmm4, %xmm15 -; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm13, %xmm1 +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm6[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm14[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,5,7,6,4] +; SSE-NEXT: pand %xmm2, %xmm14 +; SSE-NEXT: por %xmm1, %xmm14 +; SSE-NEXT: packuswb %xmm14, %xmm1 +; SSE-NEXT: movdqa %xmm10, %xmm14 +; SSE-NEXT: pandn %xmm1, %xmm14 +; SSE-NEXT: pand %xmm10, %xmm4 +; SSE-NEXT: por %xmm4, %xmm14 +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm12, %xmm1 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm13 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3],xmm13[4],xmm5[4],xmm13[5],xmm5[5],xmm13[6],xmm5[6],xmm13[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm13[2,2,3,3] +; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm5[0],xmm12[1],xmm5[1],xmm12[2],xmm5[2],xmm12[3],xmm5[3],xmm12[4],xmm5[4],xmm12[5],xmm5[5],xmm12[6],xmm5[6],xmm12[7],xmm5[7] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm12[2,2,3,3] ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3] ; SSE-NEXT: psrld $16, %xmm1 -; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm13[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,5,7,6,7] -; SSE-NEXT: punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm1[2],xmm13[3],xmm1[3] -; SSE-NEXT: packuswb %xmm13, %xmm4 -; SSE-NEXT: movdqa %xmm9, %xmm15 -; SSE-NEXT: movdqa %xmm9, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm12[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,5,7,6,7] +; SSE-NEXT: punpckhdq {{.*#+}} xmm12 = xmm12[2],xmm1[2],xmm12[3],xmm1[3] +; SSE-NEXT: packuswb %xmm12, %xmm4 +; SSE-NEXT: movdqa %xmm8, %xmm14 +; SSE-NEXT: movdqa %xmm8, %xmm1 ; SSE-NEXT: pandn %xmm4, %xmm1 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload ; SSE-NEXT: movdqa %xmm6, %xmm4 @@ -2720,46 +2696,46 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,1,0,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,1,1,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,7,6,7] -; SSE-NEXT: movdqa %xmm0, %xmm13 -; SSE-NEXT: pandn %xmm4, %xmm13 +; SSE-NEXT: movdqa %xmm0, %xmm12 +; SSE-NEXT: pandn %xmm4, %xmm12 ; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm6[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,3,2,0,4,5,6,7] ; SSE-NEXT: pand %xmm0, %xmm4 -; SSE-NEXT: por %xmm13, %xmm4 +; SSE-NEXT: por %xmm12, %xmm4 ; SSE-NEXT: packuswb %xmm4, %xmm4 -; SSE-NEXT: pand %xmm9, %xmm4 +; SSE-NEXT: pand %xmm8, %xmm4 ; SSE-NEXT: por %xmm1, %xmm4 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload ; SSE-NEXT: movdqa %xmm6, %xmm1 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] -; SSE-NEXT: movdqa %xmm2, %xmm13 -; SSE-NEXT: pandn %xmm1, %xmm13 +; SSE-NEXT: movdqa %xmm2, %xmm12 +; SSE-NEXT: pandn %xmm1, %xmm12 ; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm6[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,7,6,4] ; SSE-NEXT: pand %xmm2, %xmm1 -; SSE-NEXT: por %xmm13, %xmm1 +; SSE-NEXT: por %xmm12, %xmm1 ; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa %xmm7, %xmm13 -; SSE-NEXT: pandn %xmm1, %xmm13 -; SSE-NEXT: pand %xmm7, %xmm4 -; SSE-NEXT: por %xmm4, %xmm13 -; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm14, %xmm1 +; SSE-NEXT: movdqa %xmm10, %xmm12 +; SSE-NEXT: pandn %xmm1, %xmm12 +; SSE-NEXT: pand %xmm10, %xmm4 +; SSE-NEXT: por %xmm4, %xmm12 +; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm11, %xmm1 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3],xmm14[4],xmm5[4],xmm14[5],xmm5[5],xmm14[6],xmm5[6],xmm14[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm14[2,2,3,3] +; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm5[0],xmm11[1],xmm5[1],xmm11[2],xmm5[2],xmm11[3],xmm5[3],xmm11[4],xmm5[4],xmm11[5],xmm5[5],xmm11[6],xmm5[6],xmm11[7],xmm5[7] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm11[2,2,3,3] ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3] ; SSE-NEXT: psrld $16, %xmm1 -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm14[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,7,6,7] -; SSE-NEXT: punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm1[2],xmm9[3],xmm1[3] -; SSE-NEXT: packuswb %xmm9, %xmm4 -; SSE-NEXT: movdqa %xmm15, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm11[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,5,7,6,7] +; SSE-NEXT: punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm1[2],xmm8[3],xmm1[3] +; SSE-NEXT: packuswb %xmm8, %xmm4 +; SSE-NEXT: movdqa %xmm14, %xmm1 ; SSE-NEXT: pandn %xmm4, %xmm1 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload ; SSE-NEXT: movdqa %xmm6, %xmm4 @@ -2767,41 +2743,41 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,1,0,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,1,1,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,7,6,7] -; SSE-NEXT: movdqa %xmm0, %xmm9 -; SSE-NEXT: pandn %xmm4, %xmm9 +; SSE-NEXT: movdqa %xmm0, %xmm8 +; SSE-NEXT: pandn %xmm4, %xmm8 ; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm6[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,3,2,0,4,5,6,7] ; SSE-NEXT: pand %xmm0, %xmm4 -; SSE-NEXT: por %xmm9, %xmm4 +; SSE-NEXT: por %xmm8, %xmm4 ; SSE-NEXT: packuswb %xmm4, %xmm4 -; SSE-NEXT: pand %xmm15, %xmm4 +; SSE-NEXT: pand %xmm14, %xmm4 ; SSE-NEXT: por %xmm1, %xmm4 -; SSE-NEXT: movdqa %xmm12, %xmm1 +; SSE-NEXT: movdqa %xmm15, %xmm1 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] -; SSE-NEXT: movdqa %xmm2, %xmm9 -; SSE-NEXT: pandn %xmm1, %xmm9 -; SSE-NEXT: punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm5[8],xmm12[9],xmm5[9],xmm12[10],xmm5[10],xmm12[11],xmm5[11],xmm12[12],xmm5[12],xmm12[13],xmm5[13],xmm12[14],xmm5[14],xmm12[15],xmm5[15] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[3,1,2,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm2, %xmm8 +; SSE-NEXT: pandn %xmm1, %xmm8 +; SSE-NEXT: punpckhbw {{.*#+}} xmm15 = xmm15[8],xmm5[8],xmm15[9],xmm5[9],xmm15[10],xmm5[10],xmm15[11],xmm5[11],xmm15[12],xmm5[12],xmm15[13],xmm5[13],xmm15[14],xmm5[14],xmm15[15],xmm5[15] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm15[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,7,6,4] ; SSE-NEXT: pand %xmm2, %xmm1 -; SSE-NEXT: por %xmm9, %xmm1 +; SSE-NEXT: por %xmm8, %xmm1 ; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa %xmm7, %xmm9 -; SSE-NEXT: pandn %xmm1, %xmm9 -; SSE-NEXT: pand %xmm7, %xmm4 -; SSE-NEXT: por %xmm4, %xmm9 -; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm10, %xmm1 +; SSE-NEXT: movdqa %xmm10, %xmm8 +; SSE-NEXT: pandn %xmm1, %xmm8 +; SSE-NEXT: pand %xmm10, %xmm4 +; SSE-NEXT: por %xmm4, %xmm8 +; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm9, %xmm1 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3],xmm10[4],xmm5[4],xmm10[5],xmm5[5],xmm10[6],xmm5[6],xmm10[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm10[2,2,3,3] +; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3],xmm9[4],xmm5[4],xmm9[5],xmm5[5],xmm9[6],xmm5[6],xmm9[7],xmm5[7] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm9[2,2,3,3] ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3] ; SSE-NEXT: psrld $16, %xmm1 -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm10[0,1,0,3] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm9[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,7,6,7] ; SSE-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm1[2],xmm6[3],xmm1[3] ; SSE-NEXT: packuswb %xmm6, %xmm4 @@ -2818,32 +2794,34 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pandn %xmm1, %xmm0 ; SSE-NEXT: por %xmm3, %xmm0 ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa %xmm15, %xmm1 -; SSE-NEXT: pand %xmm15, %xmm0 +; SSE-NEXT: movdqa %xmm14, %xmm1 +; SSE-NEXT: pand %xmm14, %xmm0 ; SSE-NEXT: pandn %xmm4, %xmm1 ; SSE-NEXT: por %xmm1, %xmm0 -; SSE-NEXT: movdqa %xmm8, %xmm1 +; SSE-NEXT: movdqa %xmm7, %xmm1 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] -; SSE-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm8[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm5[8],xmm7[9],xmm5[9],xmm7[10],xmm5[10],xmm7[11],xmm5[11],xmm7[12],xmm5[12],xmm7[13],xmm5[13],xmm7[14],xmm5[14],xmm7[15],xmm5[15] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm7[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,7,6,4] ; SSE-NEXT: pand %xmm2, %xmm3 ; SSE-NEXT: pandn %xmm1, %xmm2 ; SSE-NEXT: por %xmm3, %xmm2 ; SSE-NEXT: packuswb %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm7, %xmm2 +; SSE-NEXT: movdqa %xmm10, %xmm2 ; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: pand %xmm7, %xmm0 +; SSE-NEXT: pand %xmm10, %xmm0 +; SSE-NEXT: movdqa %xmm10, %xmm11 ; SSE-NEXT: por %xmm0, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{.*#+}} xmm12 = [65535,0,65535,65535,0,65535,65535,0] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; SSE-NEXT: pand %xmm12, %xmm0 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm11, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm14 = [65535,0,65535,65535,0,65535,65535,0] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload +; SSE-NEXT: pand %xmm14, %xmm4 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm4, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,7,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,3,4,5,6,7] @@ -2853,719 +2831,712 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: packuswb %xmm1, %xmm0 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm6, %xmm1 -; SSE-NEXT: pand %xmm11, %xmm1 -; SSE-NEXT: movdqa %xmm11, %xmm4 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm6[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm10, %xmm1 +; SSE-NEXT: movdqa %xmm10, %xmm12 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] -; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa {{.*#+}} xmm3 = [0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255] -; SSE-NEXT: movdqa %xmm3, %xmm2 -; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: pand %xmm3, %xmm0 -; SSE-NEXT: por %xmm0, %xmm2 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,5,5,5,5] +; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: movdqa {{.*#+}} xmm5 = [0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255] +; SSE-NEXT: movdqa %xmm5, %xmm3 +; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: pand %xmm5, %xmm0 +; SSE-NEXT: por %xmm0, %xmm3 +; SSE-NEXT: movdqa %xmm13, %xmm0 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload -; SSE-NEXT: por %xmm0, %xmm14 -; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm1[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm7, %xmm2 -; SSE-NEXT: por %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: pand %xmm12, %xmm1 -; SSE-NEXT: movdqa %xmm12, %xmm10 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm1, %xmm0 -; SSE-NEXT: movdqa %xmm1, %xmm12 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,7,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm11, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: pand %xmm11, %xmm3 +; SSE-NEXT: movdqa %xmm11, %xmm8 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload +; SSE-NEXT: pand %xmm14, %xmm13 +; SSE-NEXT: movdqa %xmm14, %xmm7 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm13, %xmm0 +; SSE-NEXT: pand %xmm10, %xmm0 +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,7,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,3,4,5,6,7] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,5,6] -; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm15, %xmm1 -; SSE-NEXT: pand %xmm11, %xmm1 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] -; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa %xmm3, %xmm2 -; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: pand %xmm3, %xmm0 -; SSE-NEXT: por %xmm0, %xmm2 +; SSE-NEXT: packuswb %xmm2, %xmm0 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm14[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm10, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,0,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5] +; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: movdqa %xmm5, %xmm3 +; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: pand %xmm5, %xmm0 +; SSE-NEXT: por %xmm0, %xmm3 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; SSE-NEXT: por %xmm0, %xmm9 -; SSE-NEXT: movdqa %xmm9, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload +; SSE-NEXT: por %xmm0, %xmm11 +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm11[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm7, %xmm2 -; SSE-NEXT: por %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload -; SSE-NEXT: pand %xmm10, %xmm8 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm8, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,7,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm8, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: pand %xmm8, %xmm3 +; SSE-NEXT: movdqa %xmm8, %xmm9 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload +; SSE-NEXT: pand %xmm7, %xmm10 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: pand %xmm12, %xmm0 +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,7,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,3,4,5,6,7] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,5,6] -; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm13, %xmm1 -; SSE-NEXT: pand %xmm11, %xmm1 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] -; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa %xmm3, %xmm2 -; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: pand %xmm3, %xmm0 -; SSE-NEXT: por %xmm0, %xmm2 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; SSE-NEXT: packuswb %xmm2, %xmm0 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm1, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm12, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,0,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5] +; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: movdqa %xmm5, %xmm3 +; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: pand %xmm5, %xmm0 +; SSE-NEXT: por %xmm0, %xmm3 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload +; SSE-NEXT: por %xmm0, %xmm8 +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm8[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] +; SSE-NEXT: pand %xmm12, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm7, %xmm2 -; SSE-NEXT: por %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload -; SSE-NEXT: pand %xmm10, %xmm7 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,7,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm9, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: pand %xmm9, %xmm3 +; SSE-NEXT: movdqa %xmm9, %xmm1 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; SSE-NEXT: pand %xmm7, %xmm0 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm12, %xmm0 +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,7,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,3,4,5,6,7] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,5,6] -; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm11, %xmm1 -; SSE-NEXT: pand %xmm4, %xmm1 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] -; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa %xmm3, %xmm2 -; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: pand %xmm3, %xmm0 -; SSE-NEXT: por %xmm0, %xmm2 +; SSE-NEXT: packuswb %xmm2, %xmm0 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm15[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm12, %xmm2 +; SSE-NEXT: movdqa %xmm12, %xmm9 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,0,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5] +; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: movdqa %xmm5, %xmm3 +; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: pand %xmm5, %xmm0 +; SSE-NEXT: por %xmm0, %xmm3 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload -; SSE-NEXT: por %xmm0, %xmm10 -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: pand %xmm4, %xmm0 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload +; SSE-NEXT: por %xmm0, %xmm12 +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm12[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] +; SSE-NEXT: pand %xmm9, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] -; SSE-NEXT: movdqa %xmm4, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm4, %xmm2 -; SSE-NEXT: por %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload +; SSE-NEXT: movdqa %xmm1, %xmm9 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: pand %xmm1, %xmm3 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm4, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15] -; SSE-NEXT: movdqa %xmm4, %xmm1 -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[3,0] -; SSE-NEXT: movaps %xmm0, %xmm2 -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[0,2] +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15] +; SSE-NEXT: pxor %xmm7, %xmm7 +; SSE-NEXT: movdqa %xmm4, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[3,0] +; SSE-NEXT: movaps %xmm0, %xmm3 +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[0,2] ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm4[0,0] ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm4[2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm0, %xmm1 +; SSE-NEXT: packuswb %xmm0, %xmm2 ; SSE-NEXT: movdqa %xmm6, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1],xmm0[2],xmm7[2],xmm0[3],xmm7[3],xmm0[4],xmm7[4],xmm0[5],xmm7[5],xmm0[6],xmm7[6],xmm0[7],xmm7[7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm0[3,1,2,1,4,5,6,7] -; SSE-NEXT: movdqa {{.*#+}} xmm0 = [0,65535,65535,0,65535,65535,65535,65535] -; SSE-NEXT: movdqa %xmm0, %xmm4 -; SSE-NEXT: pandn %xmm2, %xmm4 -; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm6[0,3,2,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7] -; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: por %xmm4, %xmm2 -; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,1,4,5,6,7] +; SSE-NEXT: movdqa {{.*#+}} xmm3 = [0,65535,65535,0,65535,65535,65535,65535] ; SSE-NEXT: movdqa %xmm3, %xmm4 -; SSE-NEXT: pandn %xmm2, %xmm4 -; SSE-NEXT: pand %xmm3, %xmm1 -; SSE-NEXT: por %xmm1, %xmm4 -; SSE-NEXT: movdqa %xmm14, %xmm1 -; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,7,5,6,5] -; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535,0,65535,65535,0] -; SSE-NEXT: movdqa %xmm1, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3],xmm14[4],xmm5[4],xmm14[5],xmm5[5],xmm14[6],xmm5[6],xmm14[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm14[0,2,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,7] -; SSE-NEXT: pand %xmm1, %xmm2 -; SSE-NEXT: por %xmm6, %xmm2 -; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa {{.*#+}} xmm14 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] -; SSE-NEXT: movdqa %xmm14, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: pand %xmm14, %xmm4 -; SSE-NEXT: por %xmm4, %xmm6 -; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm12, %xmm2 -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm5[8],xmm12[9],xmm5[9],xmm12[10],xmm5[10],xmm12[11],xmm5[11],xmm12[12],xmm5[12],xmm12[13],xmm5[13],xmm12[14],xmm5[14],xmm12[15],xmm5[15] -; SSE-NEXT: movdqa %xmm12, %xmm4 -; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[3,0] -; SSE-NEXT: movaps %xmm2, %xmm6 -; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm4[0,2] -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,0],xmm12[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm12[2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm6[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,2] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm4 -; SSE-NEXT: movdqa %xmm15, %xmm2 -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,1,4,5,6,7] -; SSE-NEXT: movdqa %xmm0, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm15 = xmm15[8],xmm5[8],xmm15[9],xmm5[9],xmm15[10],xmm5[10],xmm15[11],xmm5[11],xmm15[12],xmm5[12],xmm15[13],xmm5[13],xmm15[14],xmm5[14],xmm15[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm15[0,3,2,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7] -; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: por %xmm6, %xmm2 -; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm3, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: pand %xmm3, %xmm4 -; SSE-NEXT: por %xmm4, %xmm6 +; SSE-NEXT: pandn %xmm0, %xmm4 +; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,3,2,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7] +; SSE-NEXT: pand %xmm3, %xmm0 +; SSE-NEXT: por %xmm4, %xmm0 +; SSE-NEXT: packuswb %xmm0, %xmm0 +; SSE-NEXT: movdqa %xmm5, %xmm6 +; SSE-NEXT: pandn %xmm0, %xmm6 +; SSE-NEXT: pand %xmm5, %xmm2 +; SSE-NEXT: por %xmm2, %xmm6 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm7[8],xmm0[9],xmm7[9],xmm0[10],xmm7[10],xmm0[11],xmm7[11],xmm0[12],xmm7[12],xmm0[13],xmm7[13],xmm0[14],xmm7[14],xmm0[15],xmm7[15] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,5] +; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,65535,0,65535,65535,0] +; SSE-NEXT: movdqa %xmm4, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[1],xmm7[1],xmm1[2],xmm7[2],xmm1[3],xmm7[3],xmm1[4],xmm7[4],xmm1[5],xmm7[5],xmm1[6],xmm7[6],xmm1[7],xmm7[7] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7] +; SSE-NEXT: pand %xmm4, %xmm0 +; SSE-NEXT: por %xmm2, %xmm0 +; SSE-NEXT: packuswb %xmm0, %xmm0 ; SSE-NEXT: movdqa %xmm9, %xmm2 -; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,5] -; SSE-NEXT: movdqa %xmm1, %xmm4 -; SSE-NEXT: pandn %xmm2, %xmm4 -; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3],xmm9[4],xmm5[4],xmm9[5],xmm5[5],xmm9[6],xmm5[6],xmm9[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm9[0,2,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,7] -; SSE-NEXT: pand %xmm1, %xmm2 -; SSE-NEXT: por %xmm4, %xmm2 -; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm14, %xmm4 -; SSE-NEXT: pandn %xmm2, %xmm4 -; SSE-NEXT: pand %xmm14, %xmm6 -; SSE-NEXT: por %xmm6, %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm8, %xmm2 -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15] -; SSE-NEXT: movdqa %xmm8, %xmm4 -; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[3,0] -; SSE-NEXT: movaps %xmm2, %xmm6 -; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm4[0,2] -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,0],xmm8[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm8[2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm6[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,2] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm4 -; SSE-NEXT: movdqa %xmm13, %xmm2 -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,1,4,5,6,7] -; SSE-NEXT: movdqa %xmm0, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm5[8],xmm13[9],xmm5[9],xmm13[10],xmm5[10],xmm13[11],xmm5[11],xmm13[12],xmm5[12],xmm13[13],xmm5[13],xmm13[14],xmm5[14],xmm13[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm13[0,3,2,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7] -; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: pand %xmm9, %xmm6 ; SSE-NEXT: por %xmm6, %xmm2 -; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm13, %xmm0 +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm1[8],xmm13[9],xmm1[9],xmm13[10],xmm1[10],xmm13[11],xmm1[11],xmm13[12],xmm1[12],xmm13[13],xmm1[13],xmm13[14],xmm1[14],xmm13[15],xmm1[15] +; SSE-NEXT: movdqa %xmm13, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[3,0] +; SSE-NEXT: movaps %xmm0, %xmm6 +; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm2[0,2] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm13[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm13[2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm6[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm14, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,1,4,5,6,7] ; SSE-NEXT: movdqa %xmm3, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: pand %xmm3, %xmm4 -; SSE-NEXT: por %xmm4, %xmm6 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload -; SSE-NEXT: movdqa %xmm8, %xmm2 -; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,5] -; SSE-NEXT: movdqa %xmm1, %xmm4 -; SSE-NEXT: pandn %xmm2, %xmm4 -; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3],xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm8[0,2,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,7] -; SSE-NEXT: pand %xmm1, %xmm2 -; SSE-NEXT: por %xmm4, %xmm2 -; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm14, %xmm4 -; SSE-NEXT: pandn %xmm2, %xmm4 -; SSE-NEXT: pand %xmm14, %xmm6 -; SSE-NEXT: por %xmm6, %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm2 -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm5[8],xmm7[9],xmm5[9],xmm7[10],xmm5[10],xmm7[11],xmm5[11],xmm7[12],xmm5[12],xmm7[13],xmm5[13],xmm7[14],xmm5[14],xmm7[15],xmm5[15] -; SSE-NEXT: movdqa %xmm7, %xmm4 -; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[3,0] -; SSE-NEXT: movaps %xmm2, %xmm6 -; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm4[0,2] -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,0],xmm7[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm7[2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm6[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,2] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm4 -; SSE-NEXT: movdqa %xmm11, %xmm2 -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,1,4,5,6,7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm5[8],xmm11[9],xmm5[9],xmm11[10],xmm5[10],xmm11[11],xmm5[11],xmm11[12],xmm5[12],xmm11[13],xmm5[13],xmm11[14],xmm5[14],xmm11[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm11[0,3,2,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,3,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,7,7,7] -; SSE-NEXT: pand %xmm0, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm0 +; SSE-NEXT: pandn %xmm0, %xmm6 +; SSE-NEXT: punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm1[8],xmm14[9],xmm1[9],xmm14[10],xmm1[10],xmm14[11],xmm1[11],xmm14[12],xmm1[12],xmm14[13],xmm1[13],xmm14[14],xmm1[14],xmm14[15],xmm1[15] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm14[0,3,2,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7] +; SSE-NEXT: pand %xmm3, %xmm0 ; SSE-NEXT: por %xmm6, %xmm0 -; SSE-NEXT: pand %xmm3, %xmm4 ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pandn %xmm0, %xmm3 -; SSE-NEXT: por %xmm4, %xmm3 -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm5[8],xmm0[9],xmm5[9],xmm0[10],xmm5[10],xmm0[11],xmm5[11],xmm0[12],xmm5[12],xmm0[13],xmm5[13],xmm0[14],xmm5[14],xmm0[15],xmm5[15] +; SSE-NEXT: movdqa %xmm5, %xmm6 +; SSE-NEXT: pandn %xmm0, %xmm6 +; SSE-NEXT: pand %xmm5, %xmm2 +; SSE-NEXT: por %xmm2, %xmm6 +; SSE-NEXT: movdqa %xmm11, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,5] -; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3],xmm10[4],xmm5[4],xmm10[5],xmm5[5],xmm10[6],xmm5[6],xmm10[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm10[0,2,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,7] -; SSE-NEXT: pand %xmm1, %xmm2 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: por %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm14, %xmm2 -; SSE-NEXT: pand %xmm14, %xmm3 -; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: pandn %xmm0, %xmm2 -; SSE-NEXT: por %xmm3, %xmm2 -; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2] -; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm13 = [255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255] -; SSE-NEXT: movdqa %xmm13, %xmm2 +; SSE-NEXT: movdqa %xmm4, %xmm2 ; SSE-NEXT: pandn %xmm0, %xmm2 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm11, %xmm0 -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pand %xmm13, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm1[0],xmm11[1],xmm1[1],xmm11[2],xmm1[2],xmm11[3],xmm1[3],xmm11[4],xmm1[4],xmm11[5],xmm1[5],xmm11[6],xmm1[6],xmm11[7],xmm1[7] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm11[0,2,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7] +; SSE-NEXT: pand %xmm4, %xmm0 ; SSE-NEXT: por %xmm2, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; SSE-NEXT: movdqa {{.*#+}} xmm6 = [65535,0,65535,65535,0,65535,65535,0] -; SSE-NEXT: pand %xmm6, %xmm9 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload +; SSE-NEXT: packuswb %xmm0, %xmm0 ; SSE-NEXT: movdqa %xmm9, %xmm2 -; SSE-NEXT: pand %xmm8, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,1,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7] -; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,65535,65535,0,0,0] -; SSE-NEXT: movdqa %xmm4, %xmm1 -; SSE-NEXT: pandn %xmm2, %xmm1 -; SSE-NEXT: pand %xmm4, %xmm0 -; SSE-NEXT: por %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm12, %xmm0 -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2] -; SSE-NEXT: packuswb %xmm2, %xmm0 -; SSE-NEXT: movdqa %xmm13, %xmm2 ; SSE-NEXT: pandn %xmm0, %xmm2 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm3, %xmm0 -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7] +; SSE-NEXT: pand %xmm9, %xmm6 +; SSE-NEXT: por %xmm6, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm1[8],xmm10[9],xmm1[9],xmm10[10],xmm1[10],xmm10[11],xmm1[11],xmm10[12],xmm1[12],xmm10[13],xmm1[13],xmm10[14],xmm1[14],xmm10[15],xmm1[15] +; SSE-NEXT: movdqa %xmm10, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[3,0] +; SSE-NEXT: movaps %xmm0, %xmm6 +; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm2[0,2] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm10[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm10[2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm6[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm0, %xmm2 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload +; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,1,4,5,6,7] +; SSE-NEXT: movdqa %xmm3, %xmm6 +; SSE-NEXT: pandn %xmm0, %xmm6 +; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm1[8],xmm7[9],xmm1[9],xmm7[10],xmm1[10],xmm7[11],xmm1[11],xmm7[12],xmm1[12],xmm7[13],xmm1[13],xmm7[14],xmm1[14],xmm7[15],xmm1[15] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,3,2,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7] +; SSE-NEXT: pand %xmm3, %xmm0 +; SSE-NEXT: por %xmm6, %xmm0 ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pand %xmm13, %xmm0 -; SSE-NEXT: por %xmm2, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: pand %xmm6, %xmm1 -; SSE-NEXT: movdqa %xmm6, %xmm7 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm1, %xmm2 -; SSE-NEXT: pand %xmm8, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,1,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7] -; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm4, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 +; SSE-NEXT: movdqa %xmm5, %xmm6 +; SSE-NEXT: pandn %xmm0, %xmm6 +; SSE-NEXT: pand %xmm5, %xmm2 +; SSE-NEXT: por %xmm2, %xmm6 +; SSE-NEXT: movdqa %xmm8, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,5] +; SSE-NEXT: movdqa %xmm4, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm1[0],xmm8[1],xmm1[1],xmm8[2],xmm1[2],xmm8[3],xmm1[3],xmm8[4],xmm1[4],xmm8[5],xmm1[5],xmm8[6],xmm1[6],xmm8[7],xmm1[7] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm8[0,2,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7] ; SSE-NEXT: pand %xmm4, %xmm0 -; SSE-NEXT: por %xmm0, %xmm6 -; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: por %xmm2, %xmm0 +; SSE-NEXT: packuswb %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm9, %xmm0 +; SSE-NEXT: pandn %xmm2, %xmm0 +; SSE-NEXT: pand %xmm9, %xmm6 +; SSE-NEXT: por %xmm6, %xmm0 +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] +; SSE-NEXT: movdqa %xmm0, %xmm6 +; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,0],xmm2[3,0] +; SSE-NEXT: movaps %xmm2, %xmm7 +; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,1],xmm6[0,2] +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,0],xmm0[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm7[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,2] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm2, %xmm6 +; SSE-NEXT: movdqa %xmm15, %xmm2 +; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,1,4,5,6,7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm15 = xmm15[8],xmm1[8],xmm15[9],xmm1[9],xmm15[10],xmm1[10],xmm15[11],xmm1[11],xmm15[12],xmm1[12],xmm15[13],xmm1[13],xmm15[14],xmm1[14],xmm15[15],xmm1[15] +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm15[0,3,2,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,3,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,7,7,7,7] +; SSE-NEXT: pand %xmm3, %xmm7 +; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: por %xmm7, %xmm3 +; SSE-NEXT: pand %xmm5, %xmm6 +; SSE-NEXT: packuswb %xmm3, %xmm3 +; SSE-NEXT: pandn %xmm3, %xmm5 +; SSE-NEXT: por %xmm6, %xmm5 +; SSE-NEXT: movdqa %xmm12, %xmm2 +; SSE-NEXT: pxor %xmm0, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,5] +; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm0[0],xmm12[1],xmm0[1],xmm12[2],xmm0[2],xmm12[3],xmm0[3],xmm12[4],xmm0[4],xmm12[5],xmm0[5],xmm12[6],xmm0[6],xmm12[7],xmm0[7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm12[0,2,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,7,7] +; SSE-NEXT: pand %xmm4, %xmm3 +; SSE-NEXT: pandn %xmm2, %xmm4 +; SSE-NEXT: por %xmm3, %xmm4 +; SSE-NEXT: pand %xmm9, %xmm5 +; SSE-NEXT: packuswb %xmm4, %xmm2 +; SSE-NEXT: pandn %xmm2, %xmm9 +; SSE-NEXT: por %xmm5, %xmm9 +; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,1,2,3] +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm10, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2] -; SSE-NEXT: packuswb %xmm2, %xmm0 -; SSE-NEXT: movdqa %xmm13, %xmm2 -; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2] +; SSE-NEXT: packuswb %xmm2, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm15 = [255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255] +; SSE-NEXT: movdqa %xmm15, %xmm2 +; SSE-NEXT: pandn %xmm1, %xmm2 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm12[3,1,2,0] +; SSE-NEXT: pand %xmm10, %xmm1 +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,0,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm1[2,1,0,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm6, %xmm6 +; SSE-NEXT: pand %xmm15, %xmm6 +; SSE-NEXT: por %xmm2, %xmm6 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload +; SSE-NEXT: movdqa {{.*#+}} xmm11 = [65535,0,65535,65535,0,65535,65535,0] +; SSE-NEXT: pand %xmm11, %xmm13 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm13[0,2,1,3] +; SSE-NEXT: pand %xmm10, %xmm1 +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,1,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7] +; SSE-NEXT: packuswb %xmm1, %xmm2 +; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535,65535,0,0,0] +; SSE-NEXT: movdqa %xmm1, %xmm3 +; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: pand %xmm1, %xmm6 +; SSE-NEXT: por %xmm6, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pand %xmm13, %xmm0 -; SSE-NEXT: por %xmm2, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload -; SSE-NEXT: pand %xmm7, %xmm15 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm15, %xmm2 -; SSE-NEXT: pand %xmm8, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3] +; SSE-NEXT: movdqa %xmm14, %xmm2 +; SSE-NEXT: pand %xmm10, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm2[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] +; SSE-NEXT: packuswb %xmm6, %xmm2 +; SSE-NEXT: movdqa %xmm15, %xmm6 +; SSE-NEXT: pandn %xmm2, %xmm6 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[3,1,2,0] +; SSE-NEXT: pand %xmm10, %xmm2 +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,1,0,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm2[2,1,0,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm7, %xmm7 +; SSE-NEXT: pand %xmm15, %xmm7 +; SSE-NEXT: por %xmm6, %xmm7 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; SSE-NEXT: pand %xmm11, %xmm3 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,1,3] +; SSE-NEXT: pand %xmm10, %xmm2 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,1,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7] -; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm4, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: pand %xmm4, %xmm0 -; SSE-NEXT: por %xmm0, %xmm6 +; SSE-NEXT: packuswb %xmm2, %xmm6 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: pandn %xmm6, %xmm2 +; SSE-NEXT: pand %xmm1, %xmm7 +; SSE-NEXT: por %xmm7, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2] -; SSE-NEXT: packuswb %xmm2, %xmm0 -; SSE-NEXT: movdqa %xmm13, %xmm2 -; SSE-NEXT: pandn %xmm0, %xmm2 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm0[2,1,0,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm6, %xmm6 -; SSE-NEXT: pand %xmm13, %xmm6 -; SSE-NEXT: por %xmm2, %xmm6 -; SSE-NEXT: movdqa %xmm7, %xmm2 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; SSE-NEXT: pand %xmm7, %xmm0 -; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload -; SSE-NEXT: por %xmm0, %xmm2 +; SSE-NEXT: pand %xmm10, %xmm6 +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm6[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,7,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,2] +; SSE-NEXT: packuswb %xmm7, %xmm6 +; SSE-NEXT: movdqa %xmm15, %xmm7 +; SSE-NEXT: pandn %xmm6, %xmm7 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm8, %xmm0 -; SSE-NEXT: pand %xmm2, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7] -; SSE-NEXT: packuswb %xmm0, %xmm2 -; SSE-NEXT: movdqa %xmm4, %xmm7 -; SSE-NEXT: pandn %xmm2, %xmm7 -; SSE-NEXT: pand %xmm4, %xmm6 -; SSE-NEXT: por %xmm6, %xmm7 -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: movdqa %xmm10, %xmm2 -; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7] -; SSE-NEXT: movdqa %xmm0, %xmm6 -; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm2[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[2,3] -; SSE-NEXT: psrlq $48, %xmm2 -; SSE-NEXT: psrldq {{.*#+}} xmm6 = xmm6[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; SSE-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm0[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,5,7] -; SSE-NEXT: packuswb %xmm6, %xmm2 -; SSE-NEXT: movdqa %xmm13, %xmm8 -; SSE-NEXT: pandn %xmm2, %xmm8 +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm2[3,1,2,0] +; SSE-NEXT: pand %xmm10, %xmm6 +; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,6,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[2,1,0,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm6[2,1,0,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm8, %xmm8 +; SSE-NEXT: pand %xmm15, %xmm8 +; SSE-NEXT: por %xmm7, %xmm8 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: pand %xmm11, %xmm2 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm2[0,2,1,3] +; SSE-NEXT: pand %xmm10, %xmm6 +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,2,1,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,1,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,0,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,6,5,4,7] +; SSE-NEXT: packuswb %xmm6, %xmm7 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: pandn %xmm7, %xmm2 +; SSE-NEXT: pand %xmm1, %xmm8 +; SSE-NEXT: por %xmm8, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa (%rsp), %xmm7 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm7, (%rsp) # 16-byte Spill +; SSE-NEXT: pand %xmm10, %xmm7 +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm7[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,7,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,1,0,2] +; SSE-NEXT: packuswb %xmm8, %xmm7 +; SSE-NEXT: movdqa %xmm15, %xmm8 +; SSE-NEXT: pandn %xmm7, %xmm8 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm6[3,1,2,0] +; SSE-NEXT: pand %xmm10, %xmm7 +; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[2,1,0,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm7[2,1,0,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm9, %xmm9 +; SSE-NEXT: pand %xmm15, %xmm9 +; SSE-NEXT: por %xmm8, %xmm9 ; SSE-NEXT: movdqa %xmm11, %xmm2 -; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5] -; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,0,65535,65535,0,65535,65535,65535] -; SSE-NEXT: movdqa %xmm0, %xmm10 -; SSE-NEXT: pandn %xmm2, %xmm10 -; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm5[0],xmm11[1],xmm5[1],xmm11[2],xmm5[2],xmm11[3],xmm5[3],xmm11[4],xmm5[4],xmm11[5],xmm5[5],xmm11[6],xmm5[6],xmm11[7],xmm5[7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm11[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm2[3,1,1,2,4,5,6,7] -; SSE-NEXT: pand %xmm0, %xmm6 -; SSE-NEXT: por %xmm10, %xmm6 -; SSE-NEXT: packuswb %xmm6, %xmm6 -; SSE-NEXT: pand %xmm13, %xmm6 -; SSE-NEXT: por %xmm8, %xmm6 -; SSE-NEXT: movdqa %xmm9, %xmm2 -; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm5[8],xmm9[9],xmm5[9],xmm9[10],xmm5[10],xmm9[11],xmm5[11],xmm9[12],xmm5[12],xmm9[13],xmm5[13],xmm9[14],xmm5[14],xmm9[15],xmm5[15] -; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm9[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,5,5,7,4] -; SSE-NEXT: movdqa {{.*#+}} xmm11 = [65535,65535,65535,65535,0,65535,0,0] -; SSE-NEXT: movdqa %xmm11, %xmm10 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload +; SSE-NEXT: pand %xmm11, %xmm7 +; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload +; SSE-NEXT: por %xmm7, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm2[0,2,1,3] +; SSE-NEXT: pand %xmm10, %xmm7 +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,2,1,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,2,1,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,0,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7] +; SSE-NEXT: packuswb %xmm7, %xmm8 +; SSE-NEXT: movdqa %xmm1, %xmm7 +; SSE-NEXT: pandn %xmm8, %xmm7 +; SSE-NEXT: pand %xmm1, %xmm9 +; SSE-NEXT: por %xmm9, %xmm7 +; SSE-NEXT: movdqa %xmm0, %xmm8 +; SSE-NEXT: pxor %xmm5, %xmm5 +; SSE-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7] +; SSE-NEXT: movdqa %xmm0, %xmm9 +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm8[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm8[2,3] +; SSE-NEXT: psrlq $48, %xmm8 +; SSE-NEXT: psrldq {{.*#+}} xmm9 = xmm9[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3] +; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm0[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,5,7] +; SSE-NEXT: packuswb %xmm9, %xmm8 +; SSE-NEXT: movdqa %xmm15, %xmm10 ; SSE-NEXT: pandn %xmm8, %xmm10 -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,1,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7] -; SSE-NEXT: pand %xmm11, %xmm2 -; SSE-NEXT: por %xmm2, %xmm10 -; SSE-NEXT: packuswb %xmm10, %xmm8 -; SSE-NEXT: movdqa %xmm4, %xmm2 -; SSE-NEXT: pandn %xmm8, %xmm2 -; SSE-NEXT: pand %xmm4, %xmm6 -; SSE-NEXT: por %xmm6, %xmm2 -; SSE-NEXT: movdqa %xmm12, %xmm9 -; SSE-NEXT: movdqa %xmm12, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3],xmm9[4],xmm5[4],xmm9[5],xmm5[5],xmm9[6],xmm5[6],xmm9[7],xmm5[7] -; SSE-NEXT: movdqa %xmm9, %xmm8 -; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[1,0],xmm6[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,0],xmm6[2,3] -; SSE-NEXT: psrlq $48, %xmm6 -; SSE-NEXT: psrldq {{.*#+}} xmm8 = xmm8[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; SSE-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm9[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,5,7] -; SSE-NEXT: packuswb %xmm8, %xmm6 +; SSE-NEXT: movdqa %xmm12, %xmm8 +; SSE-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15] +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,5,5,5,5] +; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,0,65535,65535,0,65535,65535,65535] +; SSE-NEXT: movdqa %xmm0, %xmm11 +; SSE-NEXT: pandn %xmm8, %xmm11 +; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm5[0],xmm12[1],xmm5[1],xmm12[2],xmm5[2],xmm12[3],xmm5[3],xmm12[4],xmm5[4],xmm12[5],xmm5[5],xmm12[6],xmm5[6],xmm12[7],xmm5[7] +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm12[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm8[3,1,1,2,4,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm12 +; SSE-NEXT: por %xmm11, %xmm12 +; SSE-NEXT: packuswb %xmm12, %xmm12 +; SSE-NEXT: pand %xmm15, %xmm12 +; SSE-NEXT: por %xmm10, %xmm12 ; SSE-NEXT: movdqa %xmm13, %xmm8 -; SSE-NEXT: pandn %xmm6, %xmm8 -; SSE-NEXT: movdqa %xmm3, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,5,5,5] -; SSE-NEXT: movdqa %xmm0, %xmm10 -; SSE-NEXT: pandn %xmm6, %xmm10 -; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm3[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[3,1,1,2,4,5,6,7] -; SSE-NEXT: pand %xmm0, %xmm6 -; SSE-NEXT: por %xmm10, %xmm6 -; SSE-NEXT: packuswb %xmm6, %xmm6 -; SSE-NEXT: pand %xmm13, %xmm6 -; SSE-NEXT: por %xmm8, %xmm6 -; SSE-NEXT: movdqa %xmm1, %xmm8 -; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15] -; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm1[0,1,2,3,7,5,6,7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm5[8],xmm13[9],xmm5[9],xmm13[10],xmm5[10],xmm13[11],xmm5[11],xmm13[12],xmm5[12],xmm13[13],xmm5[13],xmm13[14],xmm5[14],xmm13[15],xmm5[15] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm13[0,1,2,3,7,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,1,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,7,4] -; SSE-NEXT: movdqa %xmm11, %xmm12 -; SSE-NEXT: pandn %xmm10, %xmm12 +; SSE-NEXT: movdqa {{.*#+}} xmm11 = [65535,65535,65535,65535,0,65535,0,0] +; SSE-NEXT: movdqa %xmm11, %xmm13 +; SSE-NEXT: pandn %xmm10, %xmm13 ; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3],xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7] ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,3,1,1] ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7] ; SSE-NEXT: pand %xmm11, %xmm8 -; SSE-NEXT: por %xmm8, %xmm12 -; SSE-NEXT: packuswb %xmm12, %xmm8 -; SSE-NEXT: movdqa %xmm4, %xmm12 -; SSE-NEXT: pandn %xmm8, %xmm12 -; SSE-NEXT: pand %xmm4, %xmm6 -; SSE-NEXT: por %xmm6, %xmm12 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; SSE-NEXT: movdqa %xmm9, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3],xmm9[4],xmm5[4],xmm9[5],xmm5[5],xmm9[6],xmm5[6],xmm9[7],xmm5[7] -; SSE-NEXT: movdqa %xmm9, %xmm8 -; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[1,0],xmm6[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,0],xmm6[2,3] -; SSE-NEXT: psrlq $48, %xmm6 -; SSE-NEXT: psrldq {{.*#+}} xmm8 = xmm8[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; SSE-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm9[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,5,7] -; SSE-NEXT: packuswb %xmm8, %xmm6 -; SSE-NEXT: movdqa %xmm13, %xmm8 -; SSE-NEXT: pandn %xmm6, %xmm8 -; SSE-NEXT: movdqa %xmm14, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,5,5,5] -; SSE-NEXT: movdqa %xmm0, %xmm9 -; SSE-NEXT: pandn %xmm6, %xmm9 -; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3],xmm14[4],xmm5[4],xmm14[5],xmm5[5],xmm14[6],xmm5[6],xmm14[7],xmm5[7] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm14[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm6[3,1,1,2,4,5,6,7] -; SSE-NEXT: pand %xmm0, %xmm10 -; SSE-NEXT: por %xmm9, %xmm10 -; SSE-NEXT: packuswb %xmm10, %xmm10 -; SSE-NEXT: pand %xmm13, %xmm10 -; SSE-NEXT: por %xmm8, %xmm10 -; SSE-NEXT: movdqa %xmm15, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm15 = xmm15[8],xmm5[8],xmm15[9],xmm5[9],xmm15[10],xmm5[10],xmm15[11],xmm5[11],xmm15[12],xmm5[12],xmm15[13],xmm5[13],xmm15[14],xmm5[14],xmm15[15],xmm5[15] -; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm15[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,5,5,7,4] -; SSE-NEXT: movdqa %xmm11, %xmm9 -; SSE-NEXT: pandn %xmm8, %xmm9 -; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,3,1,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7] -; SSE-NEXT: pand %xmm11, %xmm6 -; SSE-NEXT: por %xmm6, %xmm9 -; SSE-NEXT: packuswb %xmm9, %xmm8 -; SSE-NEXT: movdqa %xmm4, %xmm6 -; SSE-NEXT: pandn %xmm8, %xmm6 -; SSE-NEXT: pand %xmm4, %xmm10 -; SSE-NEXT: por %xmm10, %xmm6 -; SSE-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload +; SSE-NEXT: por %xmm8, %xmm13 +; SSE-NEXT: packuswb %xmm13, %xmm10 ; SSE-NEXT: movdqa %xmm1, %xmm8 -; SSE-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7] -; SSE-NEXT: movdqa %xmm1, %xmm9 -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm8[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm8[2,3] -; SSE-NEXT: psrlq $48, %xmm8 -; SSE-NEXT: psrldq {{.*#+}} xmm9 = xmm9[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; SSE-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm1[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,5,7] -; SSE-NEXT: packuswb %xmm9, %xmm8 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload +; SSE-NEXT: pandn %xmm10, %xmm8 +; SSE-NEXT: pand %xmm1, %xmm12 +; SSE-NEXT: por %xmm12, %xmm8 ; SSE-NEXT: movdqa %xmm14, %xmm9 -; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm5[8],xmm9[9],xmm5[9],xmm9[10],xmm5[10],xmm9[11],xmm5[11],xmm9[12],xmm5[12],xmm9[13],xmm5[13],xmm9[14],xmm5[14],xmm9[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,5,5,5] -; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3],xmm14[4],xmm5[4],xmm14[5],xmm5[5],xmm14[6],xmm5[6],xmm14[7],xmm5[7] -; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm14[0,1,2,3,7,5,6,7] +; SSE-NEXT: movdqa %xmm14, %xmm10 +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm5[8],xmm10[9],xmm5[9],xmm10[10],xmm5[10],xmm10[11],xmm5[11],xmm10[12],xmm5[12],xmm10[13],xmm5[13],xmm10[14],xmm5[14],xmm10[15],xmm5[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3],xmm9[4],xmm5[4],xmm9[5],xmm5[5],xmm9[6],xmm5[6],xmm9[7],xmm5[7] +; SSE-NEXT: movdqa %xmm9, %xmm12 +; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[1,0],xmm10[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,0],xmm10[2,3] +; SSE-NEXT: psrlq $48, %xmm10 +; SSE-NEXT: psrldq {{.*#+}} xmm12 = xmm12[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3] +; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm9[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,4,5,7] +; SSE-NEXT: packuswb %xmm12, %xmm10 +; SSE-NEXT: movdqa %xmm15, %xmm12 +; SSE-NEXT: pandn %xmm10, %xmm12 +; SSE-NEXT: movdqa %xmm4, %xmm10 +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm5[8],xmm10[9],xmm5[9],xmm10[10],xmm5[10],xmm10[11],xmm5[11],xmm10[12],xmm5[12],xmm10[13],xmm5[13],xmm10[14],xmm5[14],xmm10[15],xmm5[15] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5] +; SSE-NEXT: movdqa %xmm0, %xmm14 +; SSE-NEXT: pandn %xmm10, %xmm14 +; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm4[0,1,2,3,7,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[3,1,1,2,4,5,6,7] -; SSE-NEXT: pand %xmm0, %xmm10 -; SSE-NEXT: pandn %xmm9, %xmm0 -; SSE-NEXT: por %xmm10, %xmm0 +; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm10[3,1,1,2,4,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm13 +; SSE-NEXT: por %xmm14, %xmm13 +; SSE-NEXT: packuswb %xmm13, %xmm13 +; SSE-NEXT: pand %xmm15, %xmm13 +; SSE-NEXT: por %xmm12, %xmm13 +; SSE-NEXT: movdqa %xmm3, %xmm10 +; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15] +; SSE-NEXT: pshufhw {{.*#+}} xmm12 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm12[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,5,5,7,4] +; SSE-NEXT: movdqa %xmm11, %xmm14 +; SSE-NEXT: pandn %xmm12, %xmm14 +; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3],xmm10[4],xmm5[4],xmm10[5],xmm5[5],xmm10[6],xmm5[6],xmm10[7],xmm5[7] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,3,1,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7] +; SSE-NEXT: pand %xmm11, %xmm10 +; SSE-NEXT: por %xmm10, %xmm14 +; SSE-NEXT: packuswb %xmm14, %xmm10 +; SSE-NEXT: movdqa %xmm1, %xmm12 +; SSE-NEXT: pandn %xmm10, %xmm12 +; SSE-NEXT: pand %xmm1, %xmm13 +; SSE-NEXT: por %xmm13, %xmm12 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload +; SSE-NEXT: movdqa %xmm9, %xmm10 +; SSE-NEXT: pxor %xmm3, %xmm3 +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm3[8],xmm10[9],xmm3[9],xmm10[10],xmm3[10],xmm10[11],xmm3[11],xmm10[12],xmm3[12],xmm10[13],xmm3[13],xmm10[14],xmm3[14],xmm10[15],xmm3[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm3[0],xmm9[1],xmm3[1],xmm9[2],xmm3[2],xmm9[3],xmm3[3],xmm9[4],xmm3[4],xmm9[5],xmm3[5],xmm9[6],xmm3[6],xmm9[7],xmm3[7] +; SSE-NEXT: pxor %xmm4, %xmm4 +; SSE-NEXT: movdqa %xmm9, %xmm13 +; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[1,0],xmm10[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,0],xmm10[2,3] +; SSE-NEXT: psrlq $48, %xmm10 +; SSE-NEXT: psrldq {{.*#+}} xmm13 = xmm13[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: punpcklwd {{.*#+}} xmm13 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3] +; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm9[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,4,5,7] +; SSE-NEXT: packuswb %xmm13, %xmm10 +; SSE-NEXT: movdqa %xmm15, %xmm13 +; SSE-NEXT: pandn %xmm10, %xmm13 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; SSE-NEXT: movdqa %xmm3, %xmm10 +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm4[8],xmm10[9],xmm4[9],xmm10[10],xmm4[10],xmm10[11],xmm4[11],xmm10[12],xmm4[12],xmm10[13],xmm4[13],xmm10[14],xmm4[14],xmm10[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5] +; SSE-NEXT: movdqa %xmm0, %xmm9 +; SSE-NEXT: pandn %xmm10, %xmm9 +; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm10[3,1,1,2,4,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm14 +; SSE-NEXT: por %xmm9, %xmm14 +; SSE-NEXT: packuswb %xmm14, %xmm14 +; SSE-NEXT: pand %xmm15, %xmm14 +; SSE-NEXT: por %xmm13, %xmm14 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; SSE-NEXT: movdqa %xmm3, %xmm9 +; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,7,4] +; SSE-NEXT: movdqa %xmm11, %xmm13 +; SSE-NEXT: pandn %xmm10, %xmm13 +; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3],xmm9[4],xmm4[4],xmm9[5],xmm4[5],xmm9[6],xmm4[6],xmm9[7],xmm4[7] +; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,3,1,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7] +; SSE-NEXT: pand %xmm11, %xmm9 +; SSE-NEXT: por %xmm9, %xmm13 +; SSE-NEXT: packuswb %xmm13, %xmm9 +; SSE-NEXT: movdqa %xmm1, %xmm13 +; SSE-NEXT: pandn %xmm9, %xmm13 +; SSE-NEXT: pand %xmm1, %xmm14 +; SSE-NEXT: por %xmm14, %xmm13 +; SSE-NEXT: movdqa (%rsp), %xmm3 # 16-byte Reload +; SSE-NEXT: movdqa %xmm3, %xmm9 +; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] +; SSE-NEXT: movdqa %xmm3, %xmm10 +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm9[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm9[2,3] +; SSE-NEXT: psrlq $48, %xmm9 +; SSE-NEXT: psrldq {{.*#+}} xmm10 = xmm10[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,4,5,7] +; SSE-NEXT: packuswb %xmm10, %xmm9 +; SSE-NEXT: movdqa %xmm6, %xmm10 +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm5[8],xmm10[9],xmm5[9],xmm10[10],xmm5[10],xmm10[11],xmm5[11],xmm10[12],xmm5[12],xmm10[13],xmm5[13],xmm10[14],xmm5[14],xmm10[15],xmm5[15] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5] +; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7] +; SSE-NEXT: pshufhw {{.*#+}} xmm14 = xmm6[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm14[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm14[3,1,1,2,4,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm14 +; SSE-NEXT: pandn %xmm10, %xmm0 +; SSE-NEXT: por %xmm14, %xmm0 ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pand %xmm13, %xmm0 -; SSE-NEXT: pandn %xmm8, %xmm13 -; SSE-NEXT: por %xmm13, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload -; SSE-NEXT: movdqa %xmm14, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm5[8],xmm14[9],xmm5[9],xmm14[10],xmm5[10],xmm14[11],xmm5[11],xmm14[12],xmm5[12],xmm14[13],xmm5[13],xmm14[14],xmm5[14],xmm14[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,1,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7] -; SSE-NEXT: pand %xmm11, %xmm1 -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm14[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm15, %xmm0 +; SSE-NEXT: pandn %xmm9, %xmm15 +; SSE-NEXT: por %xmm15, %xmm0 +; SSE-NEXT: movdqa %xmm2, %xmm4 +; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,1,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] +; SSE-NEXT: pand %xmm11, %xmm4 +; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,7,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,7,4] ; SSE-NEXT: pandn %xmm5, %xmm11 -; SSE-NEXT: por %xmm1, %xmm11 -; SSE-NEXT: pand %xmm4, %xmm0 -; SSE-NEXT: packuswb %xmm11, %xmm1 -; SSE-NEXT: pandn %xmm1, %xmm4 -; SSE-NEXT: por %xmm0, %xmm4 +; SSE-NEXT: por %xmm4, %xmm11 +; SSE-NEXT: pand %xmm1, %xmm0 +; SSE-NEXT: packuswb %xmm11, %xmm4 +; SSE-NEXT: pandn %xmm4, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE-NEXT: movaps %xmm0, 16(%rsi) ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload @@ -3606,11 +3577,11 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE-NEXT: movaps %xmm0, (%r9) ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax -; SSE-NEXT: movdqa %xmm4, 16(%rax) -; SSE-NEXT: movdqa %xmm6, 32(%rax) +; SSE-NEXT: movdqa %xmm1, 16(%rax) +; SSE-NEXT: movdqa %xmm13, 32(%rax) ; SSE-NEXT: movdqa %xmm12, 48(%rax) -; SSE-NEXT: movdqa %xmm2, (%rax) -; SSE-NEXT: addq $808, %rsp # imm = 0x328 +; SSE-NEXT: movdqa %xmm8, (%rax) +; SSE-NEXT: addq $792, %rsp # imm = 0x318 ; SSE-NEXT: retq ; ; AVX1-ONLY-LABEL: load_i8_stride6_vf64: diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll index e6de73d602b731925eec5dee4e430d03eaedfcb2..16b5e8db76ef13c966ff79c2ade4ce802200d3f5 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll @@ -652,8 +652,7 @@ define void @store_i16_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1 ; AVX512F-NEXT: vmovdqa {{.*#+}} ymm2 = ; AVX512F-NEXT: vpermd %ymm0, %ymm2, %ymm2 -; AVX512F-NEXT: vmovdqa {{.*#+}} ymm3 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535] -; AVX512F-NEXT: vpandn %ymm2, %ymm3, %ymm2 +; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 ; AVX512F-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,ymm0[10,11],zero,zero,zero,zero,ymm0[12,13],zero,zero,zero,zero,ymm0[14,15],zero,zero,zero,zero,ymm0[16,17],zero,zero,zero,zero,ymm0[18,19],zero,zero,zero,zero ; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 ; AVX512F-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2 @@ -1218,7 +1217,7 @@ define void @store_i16_stride3_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-NEXT: vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm4[4,5,6,7] ; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = <5,5,u,6,6,u,7,7,u,8,8,u,9,9,u,10> ; AVX512F-NEXT: vpermd (%rdx), %zmm4, %zmm5 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm5 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm5 ; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm3 ; AVX512F-NEXT: vmovdqa {{.*#+}} ymm6 = [128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128,20,21,128,128] ; AVX512F-NEXT: vpshufb %ymm6, %ymm3, %ymm3 @@ -2386,7 +2385,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm18 = <5,5,u,6,6,u,7,7,u,8,8,u,9,9,u,10> ; AVX512F-NEXT: vpermd 64(%rdx), %zmm18, %zmm10 ; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm22 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535] -; AVX512F-NEXT: vpternlogq $184, %zmm0, %zmm22, %zmm10 +; AVX512F-NEXT: vpternlogd $184, %zmm0, %zmm22, %zmm10 ; AVX512F-NEXT: vmovdqa 96(%rdi), %ymm0 ; AVX512F-NEXT: vmovdqa %ymm6, %ymm2 ; AVX512F-NEXT: vpshufb %ymm6, %ymm0, %ymm0 @@ -2469,7 +2468,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 ; AVX512F-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7] ; AVX512F-NEXT: vpermd (%rdx), %zmm18, %zmm1 -; AVX512F-NEXT: vpternlogq $184, %zmm0, %zmm22, %zmm1 +; AVX512F-NEXT: vpternlogd $184, %zmm0, %zmm22, %zmm1 ; AVX512F-NEXT: vmovdqa64 %zmm1, 64(%rcx) ; AVX512F-NEXT: vmovdqa64 %zmm6, 128(%rcx) ; AVX512F-NEXT: vmovdqa64 %zmm5, 192(%rcx) diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll index 9f479b0ee99377f2e538af170a049451e702a066..413cd7d63d8c67c116483084e18e115ab46c2508 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll @@ -714,7 +714,7 @@ define void @store_i16_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpbroadcastq (%r8), %ymm6 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,1,1] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm6, %zmm4 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm4 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm4 ; AVX512F-SLOW-NEXT: vpsrlq $48, %xmm3, %xmm3 ; AVX512F-SLOW-NEXT: vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm3[1] ; AVX512F-SLOW-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] @@ -753,7 +753,7 @@ define void @store_i16_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpbroadcastq (%r8), %ymm6 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,1,1] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm4, %zmm6, %zmm4 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm4 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm4 ; AVX512F-FAST-NEXT: vpsrlq $48, %xmm1, %xmm1 ; AVX512F-FAST-NEXT: vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] ; AVX512F-FAST-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] @@ -1396,7 +1396,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastq (%r8), %ymm6 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm7 = ymm4[0,1,1,1] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,14,15,u,u,u,u,u,u,u,u,16,17,u,u,u,u,u,u,u,u,18,19,u,u] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm7 = ymm0[1,1,2,2] ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2,3],ymm5[4],ymm7[5],ymm5[6],ymm7[7,8],ymm5[9],ymm7[10,11],ymm5[12],ymm7[13],ymm5[14],ymm7[15] @@ -1474,7 +1474,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastq (%r8), %ymm6 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm7 = ymm4[0,1,1,1] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,14,15,u,u,u,u,u,u,u,u,16,17,u,u,u,u,u,u,u,u,18,19,u,u] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm7 = ymm0[1,1,2,2] ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2,3],ymm5[4],ymm7[5],ymm5[6],ymm7[7,8],ymm5[9],ymm7[10,11],ymm5[12],ymm7[13],ymm5[14],ymm7[15] @@ -2776,7 +2776,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm13 = [4,5,2,3,2,3,0,1,10,11,8,9,4,5,6,7] ; AVX512F-SLOW-NEXT: vpshufb %xmm13, %xmm0, %xmm0 ; AVX512F-SLOW-NEXT: vmovdqa64 (%rdx), %ymm17 -; AVX512F-SLOW-NEXT: vmovdqa 32(%rdx), %ymm5 +; AVX512F-SLOW-NEXT: vmovdqa 32(%rdx), %ymm6 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm2 = ymm17[3,2,3,3,7,6,7,7] ; AVX512F-SLOW-NEXT: vmovdqa (%rcx), %ymm1 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm3 = ymm1[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15] @@ -2793,8 +2793,8 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,5,6] ; AVX512F-SLOW-NEXT: vmovdqa64 (%rdi), %ymm20 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm3 = ymm20[2,3,2,3,6,7,6,7] -; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %ymm6 -; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm7 = ymm6[0,1,2,3,7,6,5,7,8,9,10,11,15,14,13,15] +; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %ymm5 +; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm7 = ymm5[0,1,2,3,7,6,5,7,8,9,10,11,15,14,13,15] ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm7 = ymm7[2,3,2,2,6,7,6,6] ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm7[0],ymm3[1],ymm7[2],ymm3[3],ymm7[4,5],ymm3[6],ymm7[7,8],ymm3[9],ymm7[10],ymm3[11],ymm7[12,13],ymm3[14],ymm7[15] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,2,2] @@ -2804,7 +2804,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastq 24(%r8), %ymm0 ; AVX512F-SLOW-NEXT: vpbroadcastq 32(%r8), %ymm2 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm19 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm19 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm19 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rsi), %ymm3 ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm2 = ; AVX512F-SLOW-NEXT: vpshufb %ymm2, %ymm3, %ymm0 @@ -2822,7 +2822,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm4 = ; AVX512F-SLOW-NEXT: vpshufb %ymm4, %ymm0, %ymm2 ; AVX512F-SLOW-NEXT: vmovdqa64 %ymm4, %ymm23 -; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm8 = ymm5[3,0,3,0,7,4,7,4] +; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm8 = ymm6[3,0,3,0,7,4,7,4] ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm8[0],ymm2[1],ymm8[2],ymm2[3],ymm8[4,5],ymm2[6],ymm8[7,8],ymm2[9],ymm8[10],ymm2[11],ymm8[12,13],ymm2[14],ymm8[15] ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm8 = <10,11,u,u,6,7,u,u,8,9,8,9,u,u,8,9> ; AVX512F-SLOW-NEXT: vpshufb %xmm8, %xmm10, %xmm10 @@ -2831,15 +2831,15 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm9 = ymm9[0,1,0,0] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm9, %zmm2 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm2 -; AVX512F-SLOW-NEXT: vmovdqa (%r8), %ymm10 -; AVX512F-SLOW-NEXT: vmovdqa 32(%r8), %ymm9 +; AVX512F-SLOW-NEXT: vmovdqa (%r8), %ymm9 +; AVX512F-SLOW-NEXT: vmovdqa 32(%r8), %ymm10 ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm11 = [128,128,128,128,12,13,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128] -; AVX512F-SLOW-NEXT: vpshufb %ymm11, %ymm9, %ymm4 +; AVX512F-SLOW-NEXT: vpshufb %ymm11, %ymm10, %ymm4 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535] -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm9 = ymm9[0,1,1,1] -; AVX512F-SLOW-NEXT: vpandnq %ymm9, %ymm21, %ymm9 -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm9, %zmm9 -; AVX512F-SLOW-NEXT: vpternlogq $248, %zmm21, %zmm2, %zmm9 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm10 = ymm10[0,1,1,1] +; AVX512F-SLOW-NEXT: vpandnq %ymm10, %ymm21, %ymm10 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm10, %zmm10 +; AVX512F-SLOW-NEXT: vpternlogq $248, %zmm21, %zmm2, %zmm10 ; AVX512F-SLOW-NEXT: vmovdqa (%rdx), %xmm2 ; AVX512F-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm12[0],xmm2[0],xmm12[1],xmm2[1],xmm12[2],xmm2[2],xmm12[3],xmm2[3] ; AVX512F-SLOW-NEXT: vpshufb %xmm13, %xmm4, %xmm4 @@ -2860,43 +2860,44 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm7 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535] ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm2, %zmm7, %zmm4 ; AVX512F-SLOW-NEXT: vpbroadcastq (%r8), %ymm2 -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm8 = ymm10[0,1,1,1] -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm8, %zmm2, %zmm2 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm2 -; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm4 = ymm18[0,1,2,1,4,5,6,5] -; AVX512F-SLOW-NEXT: vprolq $16, %ymm3, %ymm8 -; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm4 = ymm8[0,1],ymm4[2],ymm8[3],ymm4[4],ymm8[5,6],ymm4[7],ymm8[8,9],ymm4[10],ymm8[11],ymm4[12],ymm8[13,14],ymm4[15] -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,3,2,3] -; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm8 = ymm18[2,3,2,3,6,7,6,7] +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm8 = ymm9[0,1,1,1] +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm8, %zmm2, %zmm12 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm12 +; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm2 = ymm18[0,1,2,1,4,5,6,5] +; AVX512F-SLOW-NEXT: vprolq $16, %ymm3, %ymm4 +; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0,1],ymm2[2],ymm4[3],ymm2[4],ymm4[5,6],ymm2[7],ymm4[8,9],ymm2[10],ymm4[11],ymm2[12],ymm4[13,14],ymm2[15] +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,3,2,3] +; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm4 = ymm18[2,3,2,3,6,7,6,7] ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,7,6,5,7,8,9,10,11,15,14,13,15] ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[2,3,2,2,6,7,6,6] -; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4,5],ymm8[6],ymm3[7,8],ymm8[9],ymm3[10],ymm8[11],ymm3[12,13],ymm8[14],ymm3[15] +; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7,8],ymm4[9],ymm3[10],ymm4[11],ymm3[12,13],ymm4[14],ymm3[15] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,2,2] -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm4 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 ; AVX512F-SLOW-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25,18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25] ; AVX512F-SLOW-NEXT: # ymm3 = mem[0,1,0,1] -; AVX512F-SLOW-NEXT: vpshufb %ymm3, %ymm0, %ymm8 -; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm12 = ymm5[1,1,1,2,5,5,5,6] -; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0],ymm12[1],ymm8[2,3],ymm12[4],ymm8[5],ymm12[6],ymm8[7,8],ymm12[9],ymm8[10,11],ymm12[12],ymm8[13],ymm12[14],ymm8[15] -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[2,3,2,3] -; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm5 = ymm5[3,2,3,3,7,6,7,7] +; AVX512F-SLOW-NEXT: vpshufb %ymm3, %ymm0, %ymm4 +; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm8 = ymm6[1,1,1,2,5,5,5,6] +; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm8[1],ymm4[2,3],ymm8[4],ymm4[5],ymm8[6],ymm4[7,8],ymm8[9],ymm4[10,11],ymm8[12],ymm4[13],ymm8[14],ymm4[15] +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,3,2,3] +; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm6 = ymm6[3,2,3,3,7,6,7,7] ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15] ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,3,2,3,6,7,6,7] -; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm5[1],ymm0[2],ymm5[3,4],ymm0[5,6,7,8],ymm5[9],ymm0[10],ymm5[11,12],ymm0[13,14,15] +; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm6[1],ymm0[2],ymm6[3,4],ymm0[5,6,7,8],ymm6[9],ymm0[10],ymm6[11,12],ymm0[13,14,15] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,3,2] -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm8, %zmm0 -; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm4, %zmm7, %zmm0 -; AVX512F-SLOW-NEXT: vpbroadcastq 48(%r8), %ymm4 -; AVX512F-SLOW-NEXT: vpbroadcastq 56(%r8), %ymm5 -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm4 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm4, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm2, %zmm7, %zmm0 +; AVX512F-SLOW-NEXT: vpbroadcastq 48(%r8), %ymm2 +; AVX512F-SLOW-NEXT: vpbroadcastq 56(%r8), %ymm4 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm2, %zmm2 +; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm4 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0] +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm0, %zmm4, %zmm2 ; AVX512F-SLOW-NEXT: vmovdqa64 %ymm22, %ymm0 -; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm6, %ymm0 -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm5 = ymm20[1,1,2,2] -; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1],ymm5[2,3],ymm0[4],ymm5[5],ymm0[6],ymm5[7,8],ymm0[9],ymm5[10,11],ymm0[12],ymm5[13],ymm0[14],ymm5[15] -; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm5 = ymm20[0,1,2,1,4,5,6,5] -; AVX512F-SLOW-NEXT: vprolq $16, %ymm6, %ymm6 -; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm5 = ymm6[0,1],ymm5[2],ymm6[3],ymm5[4],ymm6[5,6],ymm5[7],ymm6[8,9],ymm5[10],ymm6[11],ymm5[12],ymm6[13,14],ymm5[15] +; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm5, %ymm0 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm6 = ymm20[1,1,2,2] +; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm6[0],ymm0[1],ymm6[2,3],ymm0[4],ymm6[5],ymm0[6],ymm6[7,8],ymm0[9],ymm6[10,11],ymm0[12],ymm6[13],ymm0[14],ymm6[15] +; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm6 = ymm20[0,1,2,1,4,5,6,5] +; AVX512F-SLOW-NEXT: vprolq $16, %ymm5, %ymm5 +; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0,1],ymm6[2],ymm5[3],ymm6[4],ymm5[5,6],ymm6[7],ymm5[8,9],ymm6[10],ymm5[11],ymm6[12],ymm5[13,14],ymm6[15] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,3,2,3] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm0 ; AVX512F-SLOW-NEXT: vmovdqa64 %ymm23, %ymm5 @@ -2909,15 +2910,15 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm5, %zmm1 ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm0, %zmm16, %zmm1 -; AVX512F-SLOW-NEXT: vpshufb %ymm11, %ymm10, %ymm0 -; AVX512F-SLOW-NEXT: vpbroadcastq 16(%r8), %ymm3 -; AVX512F-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0 +; AVX512F-SLOW-NEXT: vpbroadcastq 16(%r8), %ymm0 +; AVX512F-SLOW-NEXT: vpandn %ymm0, %ymm4, %ymm0 +; AVX512F-SLOW-NEXT: vpshufb %ymm11, %ymm9, %ymm3 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0 ; AVX512F-SLOW-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm0, 64(%r9) -; AVX512F-SLOW-NEXT: vmovdqa64 %zmm4, 256(%r9) -; AVX512F-SLOW-NEXT: vmovdqa64 %zmm2, (%r9) -; AVX512F-SLOW-NEXT: vmovdqa64 %zmm9, 192(%r9) +; AVX512F-SLOW-NEXT: vmovdqa64 %zmm2, 256(%r9) +; AVX512F-SLOW-NEXT: vmovdqa64 %zmm12, (%r9) +; AVX512F-SLOW-NEXT: vmovdqa64 %zmm10, 192(%r9) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm19, 128(%r9) ; AVX512F-SLOW-NEXT: vzeroupper ; AVX512F-SLOW-NEXT: retq @@ -2927,7 +2928,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vmovdqa 32(%rsi), %ymm5 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm1 = ; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm5, %ymm0 -; AVX512F-FAST-NEXT: vmovdqa64 %ymm1, %ymm23 +; AVX512F-FAST-NEXT: vmovdqa64 %ymm1, %ymm17 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %ymm7 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm1 = ymm7[1,1,2,2] ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15] @@ -2935,29 +2936,29 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vmovdqa 32(%rsi), %xmm2 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm12 = <6,7,u,u,10,11,6,7,u,u,8,9,u,u,12,13> ; AVX512F-FAST-NEXT: vpshufb %xmm12, %xmm2, %xmm1 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm2, %xmm19 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm2, %xmm20 ; AVX512F-FAST-NEXT: vpbroadcastq 40(%rdi), %xmm2 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3],xmm2[4],xmm1[5],xmm2[6],xmm1[7] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; AVX512F-FAST-NEXT: vmovdqa (%rcx), %ymm4 +; AVX512F-FAST-NEXT: vmovdqa (%rcx), %ymm1 ; AVX512F-FAST-NEXT: vmovdqa 32(%rcx), %ymm8 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = -; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm8, %ymm1 -; AVX512F-FAST-NEXT: vmovdqa64 %ymm2, %ymm24 -; AVX512F-FAST-NEXT: vmovdqa64 (%rdx), %ymm17 +; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm3 = +; AVX512F-FAST-NEXT: vpshufb %ymm3, %ymm8, %ymm2 +; AVX512F-FAST-NEXT: vmovdqa64 %ymm3, %ymm24 +; AVX512F-FAST-NEXT: vmovdqa64 (%rdx), %ymm18 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdx), %ymm9 -; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm2 = ymm9[3,0,3,0,7,4,7,4] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7,8],ymm1[9],ymm2[10],ymm1[11],ymm2[12,13],ymm1[14],ymm2[15] +; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm4 = ymm9[3,0,3,0,7,4,7,4] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2],ymm2[3],ymm4[4,5],ymm2[6],ymm4[7,8],ymm2[9],ymm4[10],ymm2[11],ymm4[12,13],ymm2[14],ymm4[15] ; AVX512F-FAST-NEXT: vmovdqa 32(%rcx), %xmm11 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm15 = <10,11,u,u,6,7,u,u,8,9,8,9,u,u,8,9> -; AVX512F-FAST-NEXT: vpshufb %xmm15, %xmm11, %xmm2 +; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm4 = <10,11,u,u,6,7,u,u,8,9,8,9,u,u,8,9> +; AVX512F-FAST-NEXT: vpshufb %xmm4, %xmm11, %xmm13 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdx), %xmm6 -; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm13 = xmm6[1,2,2,2] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm13[1],xmm2[2],xmm13[3],xmm2[4,5],xmm13[6],xmm2[7] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,0] -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 +; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm15 = xmm6[1,2,2,2] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0],xmm15[1],xmm13[2],xmm15[3],xmm13[4,5],xmm15[6],xmm13[7] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,1,0,0] +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm13, %zmm15 +; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm15 ; AVX512F-FAST-NEXT: vmovdqa (%r8), %ymm0 ; AVX512F-FAST-NEXT: vmovdqa 32(%r8), %ymm2 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm13 = [128,128,128,128,12,13,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128] @@ -2966,64 +2967,65 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,1,1] ; AVX512F-FAST-NEXT: vpandnq %ymm2, %ymm16, %ymm2 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm18 -; AVX512F-FAST-NEXT: vpternlogq $248, %zmm16, %zmm1, %zmm18 -; AVX512F-FAST-NEXT: vmovdqa (%rcx), %xmm1 -; AVX512F-FAST-NEXT: vpshufb %xmm15, %xmm1, %xmm3 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm19 +; AVX512F-FAST-NEXT: vpternlogq $248, %zmm16, %zmm15, %zmm19 +; AVX512F-FAST-NEXT: vmovdqa (%rcx), %xmm3 +; AVX512F-FAST-NEXT: vpshufb %xmm4, %xmm3, %xmm4 ; AVX512F-FAST-NEXT: vmovdqa (%rdx), %xmm15 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm14 = xmm15[1,2,2,2] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm14[1],xmm3[2],xmm14[3],xmm3[4,5],xmm14[6],xmm3[7] -; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm15[0],xmm1[1],xmm15[1],xmm1[2],xmm15[2],xmm1[3],xmm15[3] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm14[1],xmm4[2],xmm14[3],xmm4[4,5],xmm14[6],xmm4[7] +; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm15[0],xmm3[1],xmm15[1],xmm3[2],xmm15[2],xmm3[3],xmm15[3] ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm2 = [4,5,2,3,2,3,0,1,10,11,8,9,4,5,6,7] -; AVX512F-FAST-NEXT: vpshufb %xmm2, %xmm1, %xmm1 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm2, %xmm20 +; AVX512F-FAST-NEXT: vpshufb %xmm2, %xmm3, %xmm3 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm2, %xmm21 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm14 = [0,1,0,1,8,9,8,8] -; AVX512F-FAST-NEXT: vpermi2q %zmm3, %zmm1, %zmm14 -; AVX512F-FAST-NEXT: vpshufb %xmm12, %xmm10, %xmm1 -; AVX512F-FAST-NEXT: vpbroadcastq 8(%rdi), %xmm3 -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2,3],xmm3[4],xmm1[5],xmm3[6],xmm1[7] -; AVX512F-FAST-NEXT: vmovdqa (%rdi), %xmm3 -; AVX512F-FAST-NEXT: vmovdqa64 32(%rdi), %xmm21 -; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3] +; AVX512F-FAST-NEXT: vpermi2q %zmm4, %zmm3, %zmm14 +; AVX512F-FAST-NEXT: vpshufb %xmm12, %xmm10, %xmm3 +; AVX512F-FAST-NEXT: vpbroadcastq 8(%rdi), %xmm4 +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5],xmm4[6],xmm3[7] +; AVX512F-FAST-NEXT: vmovdqa (%rdi), %xmm4 +; AVX512F-FAST-NEXT: vmovdqa64 32(%rdi), %xmm22 +; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3] ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,8,9,10,11,4,5,4,5,6,7,12,13] -; AVX512F-FAST-NEXT: vpshufb %xmm2, %xmm3, %xmm3 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm2, %xmm22 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1 -; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm1[0,1,0,1,4,5,4,5] +; AVX512F-FAST-NEXT: vpshufb %xmm2, %xmm4, %xmm4 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm2, %xmm23 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm3 +; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm3 = zmm3[0,1,0,1,4,5,4,5] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm16 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535] -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm14, %zmm16, %zmm1 -; AVX512F-FAST-NEXT: vpbroadcastq (%r8), %ymm3 +; AVX512F-FAST-NEXT: vpternlogq $226, %zmm14, %zmm16, %zmm3 +; AVX512F-FAST-NEXT: vpbroadcastq (%r8), %ymm4 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm10 = ymm0[0,1,1,1] -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm10, %zmm3, %zmm10 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm10 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm10, %zmm4, %zmm10 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm10 ; AVX512F-FAST-NEXT: vmovdqa (%rsi), %ymm15 -; AVX512F-FAST-NEXT: vmovdqa64 %ymm23, %ymm1 -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm15, %ymm1 +; AVX512F-FAST-NEXT: vmovdqa64 %ymm17, %ymm2 +; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm15, %ymm3 ; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm14 -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm3 = ymm14[1,1,2,2] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5],ymm1[6],ymm3[7,8],ymm1[9],ymm3[10,11],ymm1[12],ymm3[13],ymm1[14],ymm3[15] -; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm3 = ymm14[0,1,2,1,4,5,6,5] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm4 = ymm14[1,1,2,2] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5],ymm3[6],ymm4[7,8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13],ymm3[14],ymm4[15] +; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm4 = ymm14[0,1,2,1,4,5,6,5] ; AVX512F-FAST-NEXT: vprolq $16, %ymm15, %ymm13 -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm3 = ymm13[0,1],ymm3[2],ymm13[3],ymm3[4],ymm13[5,6],ymm3[7],ymm13[8,9],ymm3[10],ymm13[11],ymm3[12],ymm13[13,14],ymm3[15] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,2,3] -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm3 -; AVX512F-FAST-NEXT: vmovdqa64 %ymm24, %ymm1 -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm4, %ymm1 -; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm13 = ymm17[3,0,3,0,7,4,7,4] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm13 = ymm13[0],ymm1[1],ymm13[2],ymm1[3],ymm13[4,5],ymm1[6],ymm13[7,8],ymm1[9],ymm13[10],ymm1[11],ymm13[12,13],ymm1[14],ymm13[15] -; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25,18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25] -; AVX512F-FAST-NEXT: # ymm1 = mem[0,1,0,1] -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm4, %ymm2 -; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm12 = ymm17[1,1,1,2,5,5,5,6] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm4 = ymm13[0,1],ymm4[2],ymm13[3],ymm4[4],ymm13[5,6],ymm4[7],ymm13[8,9],ymm4[10],ymm13[11],ymm4[12],ymm13[13,14],ymm4[15] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,3,2,3] +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm3 +; AVX512F-FAST-NEXT: vmovdqa64 %ymm24, %ymm2 +; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm1, %ymm4 +; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm13 = ymm18[3,0,3,0,7,4,7,4] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm13 = ymm13[0],ymm4[1],ymm13[2],ymm4[3],ymm13[4,5],ymm4[6],ymm13[7,8],ymm4[9],ymm13[10],ymm4[11],ymm13[12,13],ymm4[14],ymm13[15] +; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25,18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25] +; AVX512F-FAST-NEXT: # ymm4 = mem[0,1,0,1] +; AVX512F-FAST-NEXT: vpshufb %ymm4, %ymm1, %ymm2 +; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm12 = ymm18[1,1,1,2,5,5,5,6] ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm12[1],ymm2[2,3],ymm12[4],ymm2[5],ymm12[6],ymm2[7,8],ymm12[9],ymm2[10,11],ymm12[12],ymm2[13],ymm12[14],ymm2[15] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,3,2,3] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm13, %zmm2 -; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535] -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm3, %zmm12, %zmm2 -; AVX512F-FAST-NEXT: vmovdqa64 %ymm25, %ymm3 -; AVX512F-FAST-NEXT: vpshufb %ymm3, %ymm0, %ymm0 +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535] +; AVX512F-FAST-NEXT: vpternlogq $226, %zmm3, %zmm17, %zmm2 ; AVX512F-FAST-NEXT: vpbroadcastq 16(%r8), %ymm3 -; AVX512F-FAST-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm12 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0] +; AVX512F-FAST-NEXT: vpandn %ymm3, %ymm12, %ymm3 +; AVX512F-FAST-NEXT: vmovdqa64 %ymm25, %ymm13 +; AVX512F-FAST-NEXT: vpshufb %ymm13, %ymm0, %ymm0 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0 ; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm13 = [30,31,28,29,26,27,30,31,30,31,28,29,30,31,28,29,30,31,28,29,26,27,30,31,30,31,28,29,30,31,28,29] @@ -3040,44 +3042,44 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm8, %ymm3 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm7 = ymm9[3,2,3,3,7,6,7,7] ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm7[1],ymm3[2],ymm7[3,4],ymm3[5,6,7,8],ymm7[9],ymm3[10],ymm7[11,12],ymm3[13,14,15] -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm8, %ymm1 +; AVX512F-FAST-NEXT: vpshufb %ymm4, %ymm8, %ymm4 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm7 = ymm9[1,1,1,2,5,5,5,6] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm7[1],ymm1[2,3],ymm7[4],ymm1[5],ymm7[6],ymm1[7,8],ymm7[9],ymm1[10,11],ymm7[12],ymm1[13],ymm7[14],ymm1[15] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm7[1],ymm4[2,3],ymm7[4],ymm4[5],ymm7[6],ymm4[7,8],ymm7[9],ymm4[10,11],ymm7[12],ymm4[13],ymm7[14],ymm4[15] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm7 = [2,3,2,3,10,10,11,10] -; AVX512F-FAST-NEXT: vpermi2q %zmm3, %zmm1, %zmm7 +; AVX512F-FAST-NEXT: vpermi2q %zmm3, %zmm4, %zmm7 ; AVX512F-FAST-NEXT: vpternlogq $226, %zmm5, %zmm16, %zmm7 -; AVX512F-FAST-NEXT: vpbroadcastq 48(%r8), %ymm1 -; AVX512F-FAST-NEXT: vpbroadcastq 56(%r8), %ymm3 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm1 -; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm11[0],xmm6[0],xmm11[1],xmm6[1],xmm11[2],xmm6[2],xmm11[3],xmm6[3] -; AVX512F-FAST-NEXT: vmovdqa64 %xmm20, %xmm5 -; AVX512F-FAST-NEXT: vpshufb %xmm5, %xmm3, %xmm3 -; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm4, %ymm2 -; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm4 = ymm17[3,2,3,3,7,6,7,7] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm4[1],ymm2[2],ymm4[3,4],ymm2[5,6,7,8],ymm4[9],ymm2[10],ymm4[11,12],ymm2[13,14,15] -; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm4 = [2,2,3,2,8,9,8,9] -; AVX512F-FAST-NEXT: vpermi2q %zmm3, %zmm2, %zmm4 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm19, %xmm2 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm21, %xmm3 -; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; AVX512F-FAST-NEXT: vmovdqa64 %xmm22, %xmm3 -; AVX512F-FAST-NEXT: vpshufb %xmm3, %xmm2, %xmm2 -; AVX512F-FAST-NEXT: vpshufb %ymm13, %ymm15, %ymm3 +; AVX512F-FAST-NEXT: vpbroadcastq 48(%r8), %ymm3 +; AVX512F-FAST-NEXT: vpbroadcastq 56(%r8), %ymm4 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm3 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm7, %zmm12, %zmm3 +; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm11[0],xmm6[0],xmm11[1],xmm6[1],xmm11[2],xmm6[2],xmm11[3],xmm6[3] +; AVX512F-FAST-NEXT: vmovdqa64 %xmm21, %xmm5 +; AVX512F-FAST-NEXT: vpshufb %xmm5, %xmm4, %xmm4 +; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm1, %ymm1 +; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm2 = ymm18[3,2,3,3,7,6,7,7] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3,4],ymm1[5,6,7,8],ymm2[9],ymm1[10],ymm2[11,12],ymm1[13,14,15] +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm2 = [2,2,3,2,8,9,8,9] +; AVX512F-FAST-NEXT: vpermi2q %zmm4, %zmm1, %zmm2 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm20, %xmm1 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm22, %xmm4 +; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3] +; AVX512F-FAST-NEXT: vmovdqa64 %xmm23, %xmm4 +; AVX512F-FAST-NEXT: vpshufb %xmm4, %xmm1, %xmm1 +; AVX512F-FAST-NEXT: vpshufb %ymm13, %ymm15, %ymm4 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm5 = ymm14[2,3,2,3,6,7,6,7] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4,5],ymm5[6],ymm3[7,8],ymm5[9],ymm3[10],ymm5[11],ymm3[12,13],ymm5[14],ymm3[15] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4,5],ymm5[6],ymm4[7,8],ymm5[9],ymm4[10],ymm5[11],ymm4[12,13],ymm5[14],ymm4[15] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm5 = [2,3,2,2,8,9,8,9] -; AVX512F-FAST-NEXT: vpermi2q %zmm2, %zmm3, %zmm5 -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm4, %zmm12, %zmm5 -; AVX512F-FAST-NEXT: vpbroadcastq 24(%r8), %ymm2 -; AVX512F-FAST-NEXT: vpbroadcastq 32(%r8), %ymm3 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm2 -; AVX512F-FAST-NEXT: vmovdqa64 %zmm2, 128(%r9) -; AVX512F-FAST-NEXT: vmovdqa64 %zmm1, 256(%r9) +; AVX512F-FAST-NEXT: vpermi2q %zmm1, %zmm4, %zmm5 +; AVX512F-FAST-NEXT: vpternlogq $226, %zmm2, %zmm17, %zmm5 +; AVX512F-FAST-NEXT: vpbroadcastq 24(%r8), %ymm1 +; AVX512F-FAST-NEXT: vpbroadcastq 32(%r8), %ymm2 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm1 +; AVX512F-FAST-NEXT: vmovdqa64 %zmm1, 128(%r9) +; AVX512F-FAST-NEXT: vmovdqa64 %zmm3, 256(%r9) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm0, 64(%r9) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm10, (%r9) -; AVX512F-FAST-NEXT: vmovdqa64 %zmm18, 192(%r9) +; AVX512F-FAST-NEXT: vmovdqa64 %zmm19, 192(%r9) ; AVX512F-FAST-NEXT: vzeroupper ; AVX512F-FAST-NEXT: retq ; @@ -5924,12 +5926,12 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastq 96(%r8), %ymm2 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm18 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm31, %zmm18, %zmm1 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm31, %zmm18, %zmm1 ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm25, %zmm19, %zmm27 ; AVX512F-SLOW-NEXT: vpbroadcastq 24(%r8), %ymm2 ; AVX512F-SLOW-NEXT: vpbroadcastq 32(%r8), %ymm25 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm25, %zmm2, %zmm2 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm27, %zmm18, %zmm2 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm27, %zmm18, %zmm2 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm18 = [65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0] ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload ; AVX512F-SLOW-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm31 # 64-byte Folded Reload @@ -5972,28 +5974,28 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastq 64(%r8), %ymm18 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm8, %zmm18, %zmm8 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm22, %zmm18, %zmm8 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm22, %zmm18, %zmm8 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm28, %zmm27, %zmm22 ; AVX512F-SLOW-NEXT: vpermq $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm21 = mem[0,1,0,1,4,5,4,5] ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm22, %zmm24, %zmm21 ; AVX512F-SLOW-NEXT: vpbroadcastq (%r8), %ymm22 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm22, %zmm10 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm21, %zmm18, %zmm10 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm21, %zmm18, %zmm10 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm13, %zmm14, %zmm13 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm11, %zmm12, %zmm11 ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm13, %zmm24, %zmm11 ; AVX512F-SLOW-NEXT: vpbroadcastq 112(%r8), %ymm12 ; AVX512F-SLOW-NEXT: vpbroadcastq 120(%r8), %ymm13 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm13, %zmm12, %zmm12 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm11, %zmm16, %zmm12 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm11, %zmm16, %zmm12 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm7, %zmm15, %zmm7 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm6, %zmm4 ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm7, %zmm24, %zmm4 ; AVX512F-SLOW-NEXT: vpbroadcastq 48(%r8), %ymm6 ; AVX512F-SLOW-NEXT: vpbroadcastq 56(%r8), %ymm7 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm4, %zmm16, %zmm6 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm4, %zmm16, %zmm6 ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm30, %zmm19, %zmm9 ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm20, %zmm19, %zmm0 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm4 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535] @@ -6186,7 +6188,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastq 112(%r8), %ymm0 ; AVX512F-FAST-NEXT: vpbroadcastq 120(%r8), %ymm1 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm16 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm2, %zmm17, %zmm16 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm2, %zmm17, %zmm16 ; AVX512F-FAST-NEXT: vmovdqa 64(%rsi), %ymm8 ; AVX512F-FAST-NEXT: vpshufb %ymm12, %ymm8, %ymm0 ; AVX512F-FAST-NEXT: vmovdqa64 64(%rdi), %ymm24 @@ -6231,7 +6233,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastq 48(%r8), %ymm6 ; AVX512F-FAST-NEXT: vpbroadcastq 56(%r8), %ymm10 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm10, %zmm6, %zmm6 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm7, %zmm17, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm7, %zmm17, %zmm6 ; AVX512F-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload ; AVX512F-FAST-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm7 # 16-byte Folded Reload ; AVX512F-FAST-NEXT: # xmm7 = xmm7[0],mem[0],xmm7[1],mem[1],xmm7[2],mem[2],xmm7[3],mem[3] @@ -6267,7 +6269,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastq 96(%r8), %ymm5 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm0 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm5 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm8, %zmm5, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm8, %zmm5, %zmm0 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %xmm7 ; AVX512F-FAST-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm7 # 16-byte Folded Reload ; AVX512F-FAST-NEXT: # xmm7 = xmm7[0],mem[0],xmm7[1],mem[1],xmm7[2],mem[2],xmm7[3],mem[3] @@ -6282,7 +6284,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastq 24(%r8), %ymm4 ; AVX512F-FAST-NEXT: vpbroadcastq 32(%r8), %ymm7 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm4, %zmm4 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm8, %zmm5, %zmm4 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm8, %zmm5, %zmm4 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm5 = [65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0] ; AVX512F-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload ; AVX512F-FAST-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm3 # 64-byte Folded Reload @@ -6294,10 +6296,10 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastq 64(%r8), %ymm5 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm5, %zmm2 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm20, %zmm5, %zmm2 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm20, %zmm5, %zmm2 ; AVX512F-FAST-NEXT: vpbroadcastq (%r8), %ymm7 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm9, %zmm7, %zmm7 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm22, %zmm5, %zmm7 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm22, %zmm5, %zmm7 ; AVX512F-FAST-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm26 # 64-byte Folded Reload ; AVX512F-FAST-NEXT: vpternlogq $226, %zmm25, %zmm1, %zmm12 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535] diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll index 7133214d9a80f76a2d4b13a013425aefb54e0877..b2c0e00825e63e08d563128cf17d4102da46fd7b 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll @@ -3486,19 +3486,19 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-ONLY-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm19, %zmm18, %zmm9 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm10 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm17, %zmm10, %zmm9 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm17, %zmm10, %zmm9 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm22, %zmm21, %zmm11 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm20, %zmm10, %zmm11 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm20, %zmm10, %zmm11 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm24, %zmm16, %zmm10 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm23, %zmm12, %zmm10 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm23, %zmm12, %zmm10 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm8, %zmm3 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm12, %zmm3 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm1, %zmm12, %zmm3 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm1 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm6 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm4, %zmm6, %zmm1 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm4, %zmm6, %zmm1 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm5, %zmm2 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm6, %zmm2 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm0, %zmm6, %zmm2 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm2, (%rax) ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm1, 192(%rax) ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm3, 256(%rax) @@ -3695,23 +3695,23 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-ONLY-FAST-NEXT: vpermt2q %zmm3, %zmm5, %zmm2 ; AVX512F-ONLY-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm0[0,1,2,3] ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm0, %zmm3, %zmm2 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm0, %zmm3, %zmm2 ; AVX512F-ONLY-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm2, 256(%rax) ; AVX512F-ONLY-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm18[0,1,2,3],zmm10[0,1,2,3] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm0, %zmm3, %zmm8 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm0, %zmm3, %zmm8 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm8, 64(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm22, %zmm0, %zmm14 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm22, %zmm0, %zmm14 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm14, (%rax) -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm19, %zmm0, %zmm1 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm19, %zmm0, %zmm1 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm1, 192(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm17, %zmm0, %zmm1 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm17, %zmm0, %zmm1 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm1, 128(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm16, %zmm0, %zmm1 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm16, %zmm0, %zmm1 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm1, 320(%rax) ; AVX512F-ONLY-FAST-NEXT: popq %rax ; AVX512F-ONLY-FAST-NEXT: vzeroupper @@ -3935,19 +3935,19 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512DQ-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm19, %zmm26, %zmm6 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm7 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm6 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm6 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm18, %zmm22, %zmm8 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm21, %zmm7, %zmm8 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm21, %zmm7, %zmm8 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm25, %zmm24, %zmm7 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm9 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm20, %zmm9, %zmm7 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm20, %zmm9, %zmm7 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm10 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm23, %zmm9, %zmm10 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm23, %zmm9, %zmm10 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm3, %zmm1 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm2, %zmm3, %zmm1 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm2 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm3, %zmm2 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm0, %zmm3, %zmm2 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm2, 256(%rax) ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm1, 64(%rax) ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm8, (%rax) @@ -4154,22 +4154,22 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,0,2,1,4,5,6,7] ; AVX512DQ-FAST-NEXT: vpermt2q %zmm5, %zmm9, %zmm6 ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm4, %zmm5, %zmm6 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm4, %zmm5, %zmm6 ; AVX512DQ-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm6, (%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm3, %zmm5, %zmm7 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm3, %zmm5, %zmm7 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm7, 192(%rax) ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm3 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm1, %zmm3, %zmm0 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm1, %zmm3, %zmm0 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm0, 128(%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm18, %zmm3, %zmm2 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm18, %zmm3, %zmm2 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm2, 320(%rax) ; AVX512DQ-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm20[0,1,2,3],zmm17[0,1,2,3] ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm1, %zmm13 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm0, %zmm1, %zmm13 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm13, 256(%rax) ; AVX512DQ-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm19[0,1,2,3],zmm23[0,1,2,3] -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm1, %zmm12 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm0, %zmm1, %zmm12 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm12, 64(%rax) ; AVX512DQ-FAST-NEXT: vzeroupper ; AVX512DQ-FAST-NEXT: retq @@ -7763,10 +7763,10 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 32-byte Folded Reload ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm29 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm6 # 64-byte Folded Reload +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm6 # 64-byte Folded Reload ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm23 # 32-byte Folded Reload -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm23 # 64-byte Folded Reload +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm23 # 64-byte Folded Reload ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm22, %zmm2, %zmm12 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm14, %zmm0, %zmm14 @@ -7862,27 +7862,27 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,2,2,3,4,5,6,7] ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,4,4,4] ; AVX512F-ONLY-SLOW-NEXT: vpermq {{.*#+}} ymm11 = ymm11[0,0,2,1] -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm12 # 64-byte Folded Reload +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm12 # 64-byte Folded Reload ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm16, %zmm20, %zmm4 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm4 # 64-byte Folded Reload +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm4 # 64-byte Folded Reload ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm5, %zmm5 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm5 # 64-byte Folded Reload +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm5 # 64-byte Folded Reload ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm17, %zmm24, %zmm16 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm27, %zmm10, %zmm16 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm27, %zmm10, %zmm16 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm13, %zmm8, %zmm8 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm25, %zmm10, %zmm8 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm25, %zmm10, %zmm8 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm14, %zmm2 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm22, %zmm10, %zmm2 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm22, %zmm10, %zmm2 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm19, %zmm18, %zmm10 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm28, %zmm13, %zmm10 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm28, %zmm13, %zmm10 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm7, %zmm7 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm30, %zmm13, %zmm7 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm30, %zmm13, %zmm7 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm15, %zmm3, %zmm3 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm21, %zmm13, %zmm3 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm21, %zmm13, %zmm3 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm11, %zmm1, %zmm1 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm13, %zmm1 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm0, %zmm13, %zmm1 ; AVX512F-ONLY-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm3, 192(%rax) @@ -8295,40 +8295,40 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-ONLY-FAST-NEXT: vpermt2q %zmm7, %zmm27, %zmm6 ; AVX512F-ONLY-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm18[0,1,2,3] ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm7 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm0, %zmm7, %zmm6 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm0, %zmm7, %zmm6 ; AVX512F-ONLY-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm6, 256(%rax) ; AVX512F-ONLY-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm20[0,1,2,3] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm0, %zmm7, %zmm9 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm0, %zmm7, %zmm9 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm9, 448(%rax) ; AVX512F-ONLY-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm5[0,1,2,3],zmm28[0,1,2,3] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm0, %zmm7, %zmm8 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm0, %zmm7, %zmm8 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm8, 640(%rax) ; AVX512F-ONLY-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm2[0,1,2,3],zmm17[0,1,2,3] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm0, %zmm7, %zmm3 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm0, %zmm7, %zmm3 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm3, 64(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm16, %zmm0, %zmm12 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm16, %zmm0, %zmm12 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm12, (%rax) -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm26, %zmm0, %zmm4 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm26, %zmm0, %zmm4 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm4, 192(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm25, %zmm1, %zmm10 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm25, %zmm1, %zmm10 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm10, 128(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm2 # 64-byte Folded Reload +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm2 # 64-byte Folded Reload ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm2, 320(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm2, 384(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm2, 576(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm0, 512(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm0, 704(%rax) ; AVX512F-ONLY-FAST-NEXT: addq $1256, %rsp # imm = 0x4E8 ; AVX512F-ONLY-FAST-NEXT: vzeroupper @@ -8731,10 +8731,10 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512DQ-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 32-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm26 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm2 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm2 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 32-byte Folded Reload -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm3 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm3 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm4 # 32-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload @@ -8807,27 +8807,27 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512DQ-SLOW-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,1,0,1] ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} ymm10 = ymm10[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15] ; AVX512DQ-SLOW-NEXT: vpermq {{.*#+}} ymm10 = ymm10[2,2,2,2] -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm4 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm4 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm31, %zmm30, %zmm25 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm25 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm25 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm18, %zmm23, %zmm6 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm18 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm6 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm6 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm21, %zmm17, %zmm16 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm16 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm16 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm14, %zmm24, %zmm14 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm14 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm14 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm12, %zmm5, %zmm5 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm29, %zmm18, %zmm5 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm29, %zmm18, %zmm5 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm8, %zmm8 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm9 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm19, %zmm9, %zmm8 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm19, %zmm9, %zmm8 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm15, %zmm0 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm20, %zmm9, %zmm0 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm20, %zmm9, %zmm0 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm11, %zmm7, %zmm7 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm22, %zmm9, %zmm7 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm22, %zmm9, %zmm7 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm13, %zmm10 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm9, %zmm10 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm1, %zmm9, %zmm10 ; AVX512DQ-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm10, 256(%rax) ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm7, 448(%rax) @@ -9253,43 +9253,43 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm13 = xmm6[0,0,2,1,4,5,6,7] ; AVX512DQ-FAST-NEXT: vpermt2q %zmm11, %zmm18, %zmm13 ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm9, %zmm11, %zmm13 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm9, %zmm11, %zmm13 ; AVX512DQ-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm13, (%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm5, %zmm11, %zmm10 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm5, %zmm11, %zmm10 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm10, 192(%rax) ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm5 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm5, %zmm8 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm0, %zmm5, %zmm8 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm8, 128(%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm29, %zmm5, %zmm3 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm29, %zmm5, %zmm3 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm3, 320(%rax) ; AVX512DQ-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm20[0,1,2,3],zmm25[0,1,2,3] ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] ; AVX512DQ-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm3, %zmm6 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm0, %zmm3, %zmm6 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm6, 256(%rax) ; AVX512DQ-FAST-NEXT: vshufi64x2 $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload ; AVX512DQ-FAST-NEXT: # zmm0 = zmm17[0,1,2,3],mem[0,1,2,3] ; AVX512DQ-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm3, %zmm6 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm0, %zmm3, %zmm6 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm6, 448(%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm12, %zmm11, %zmm4 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm12, %zmm11, %zmm4 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm4, 384(%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm7 # 64-byte Folded Reload +; AVX512DQ-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm7 # 64-byte Folded Reload ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm7, 576(%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm2 # 64-byte Folded Reload +; AVX512DQ-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm2 # 64-byte Folded Reload ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm2, 512(%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm1 # 64-byte Folded Reload +; AVX512DQ-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm1 # 64-byte Folded Reload ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm1, 704(%rax) ; AVX512DQ-FAST-NEXT: vshufi64x2 $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm0 # 64-byte Folded Reload ; AVX512DQ-FAST-NEXT: # zmm0 = zmm26[0,1,2,3],mem[0,1,2,3] ; AVX512DQ-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm3, %zmm1 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm0, %zmm3, %zmm1 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm1, 640(%rax) ; AVX512DQ-FAST-NEXT: vshufi64x2 $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm22, %zmm0 # 64-byte Folded Reload ; AVX512DQ-FAST-NEXT: # zmm0 = zmm22[0,1,2,3],mem[0,1,2,3] ; AVX512DQ-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm3, %zmm1 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm0, %zmm3, %zmm1 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm1, 64(%rax) ; AVX512DQ-FAST-NEXT: addq $1224, %rsp # imm = 0x4C8 ; AVX512DQ-FAST-NEXT: vzeroupper diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll index 5934b80893ce39026cd15ce0e6cafb8de7699c6f..0bb8e17e0ee484df705ff390161c2b38ceb3b422 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll @@ -458,7 +458,7 @@ define void @store_i16_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[4,5,12,13,u,u,u,u,u,u],zero,zero,zero,zero,ymm3[6,7],zero,zero,ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 ; AVX512F-SLOW-NEXT: vporq %zmm1, %zmm2, %zmm1 -; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 +; AVX512F-SLOW-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 ; AVX512F-SLOW-NEXT: vextracti32x4 $2, %zmm1, 32(%rax) ; AVX512F-SLOW-NEXT: vextracti32x4 $3, %zmm1, %xmm0 ; AVX512F-SLOW-NEXT: vmovq %xmm0, 48(%rax) @@ -1059,7 +1059,7 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpbroadcastd 4(%r10), %ymm12 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm11 ; AVX512F-SLOW-NEXT: vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm11 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm11 ; AVX512F-SLOW-NEXT: vpsrlq $48, %xmm4, %xmm4 ; AVX512F-SLOW-NEXT: vpunpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm4[1] ; AVX512F-SLOW-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] @@ -1141,7 +1141,7 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpbroadcastd 4(%r10), %ymm5 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4 ; AVX512F-FAST-NEXT: vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm4 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm4 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm4 ; AVX512F-FAST-NEXT: vmovdqa %xmm0, 96(%rax) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm4, (%rax) ; AVX512F-FAST-NEXT: vmovdqa %ymm1, 64(%rax) @@ -2341,8 +2341,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm10 = ymm2[u,u,u,u,u,u,u,u,u,u,14,15],zero,zero,ymm2[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm2[u,u,u,u] ; AVX512F-SLOW-NEXT: vporq %ymm5, %ymm10, %ymm19 ; AVX512F-SLOW-NEXT: vpbroadcastd 8(%rax), %ymm5 -; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm10 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535] -; AVX512F-SLOW-NEXT: vpandn %ymm5, %ymm10, %ymm5 +; AVX512F-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm5 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm10 = ymm13[12,13,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm13[14,15,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm13[16,17,u,u] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm5, %zmm5 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm10 = ymm6[0,1,1,3,4,5,5,7] @@ -2427,22 +2426,22 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm16, %zmm7, %zmm7 ; AVX512F-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rcx ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm17, %zmm9, %zmm9 -; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm9 +; AVX512F-SLOW-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm9 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm19, %zmm4, %zmm4 ; AVX512F-SLOW-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm5 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm5 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm11, %zmm10, %zmm4 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm7 = zmm20[2,2,2,3,6,6,6,7] -; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm7 +; AVX512F-SLOW-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm7 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm12, %zmm0 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm3 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm3 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm3 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm0 = zmm1[0,0,1,1,4,4,5,5] -; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm18, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm18, %zmm0 ; AVX512F-SLOW-NEXT: vpbroadcastd (%rax), %ymm1 ; AVX512F-SLOW-NEXT: vpbroadcastd 4(%rax), %ymm4 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm1 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm1 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm1 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm6 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm13[2,1,3,2] @@ -2556,20 +2555,20 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rcx ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm16, %zmm3, %zmm2 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm17, %zmm0, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm18, %zmm8, %zmm2 ; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm10 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm10 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm0 = zmm19[2,2,2,3,6,6,6,7] -; AVX512F-FAST-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm13, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm13, %zmm6 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm6 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm0 = zmm11[0,0,1,1,4,4,5,5] -; AVX512F-FAST-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm0 ; AVX512F-FAST-NEXT: vpbroadcastd (%rax), %ymm2 ; AVX512F-FAST-NEXT: vpbroadcastd 4(%rax), %ymm3 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm20, %zmm2 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm20, %zmm2 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm4 ; AVX512F-FAST-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm1 @@ -5262,12 +5261,12 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm0 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm9 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm9 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm3 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm3 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm7 = ymm15[2,1,3,2] ; AVX512F-SLOW-NEXT: vpbroadcastd 32(%rax), %ymm9 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm7, %zmm7 ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm7 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm7 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm7 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm8, %zmm17, %zmm1 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm13, %zmm18, %zmm8 @@ -5276,7 +5275,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastd 40(%rax), %ymm9 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm1, %zmm1 ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm1 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm1 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm1 ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm19, %zmm8, %zmm8 @@ -5295,7 +5294,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastd 4(%rax), %ymm10 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm9, %zmm9 ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm9 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm9 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm9 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm14, %zmm2 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4 @@ -5303,7 +5302,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vbroadcasti64x4 {{.*#+}} zmm2 = [0,5,4,0,0,6,5,0,0,5,4,0,0,6,5,0] ; AVX512F-SLOW-NEXT: # zmm2 = mem[0,1,2,3,0,1,2,3] ; AVX512F-SLOW-NEXT: vpermd (%rax), %zmm2, %zmm2 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm2 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm2 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm2 ; AVX512F-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm2, 128(%rax) @@ -5541,11 +5540,11 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [30,5,0,0,31,6,0,31,30,5,0,0,31,6,0,31] ; AVX512F-FAST-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3] ; AVX512F-FAST-NEXT: vpermi2d %zmm3, %zmm8, %zmm7 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm7 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm7 ; AVX512F-FAST-NEXT: vpbroadcastd 36(%rax), %ymm6 ; AVX512F-FAST-NEXT: vpbroadcastd 40(%rax), %ymm8 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm8, %zmm6, %zmm6 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm6 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535] ; AVX512F-FAST-NEXT: vpternlogq $226, %zmm26, %zmm8, %zmm5 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 @@ -5558,7 +5557,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm11 = ymm19[0,0,1,3] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm12 = ymm18[2,2,2,3] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm13 = ymm17[2,1,3,2] -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm5 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm5 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm5 ; AVX512F-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm15[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u] @@ -5574,7 +5573,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpermd %ymm9, %ymm0, %ymm0 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm9, %zmm3, %zmm3 ; AVX512F-FAST-NEXT: vpermd %zmm3, %zmm21, %zmm3 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm28, %zmm3 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm28, %zmm3 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm14, %zmm3 ; AVX512F-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm4 # 32-byte Folded Reload ; AVX512F-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm9 # 32-byte Folded Reload @@ -5591,7 +5590,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm16 ; AVX512F-FAST-NEXT: vpbroadcastd 32(%rax), %ymm2 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm22, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm22, %zmm0 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm20, %zmm0 ; AVX512F-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-FAST-NEXT: vmovdqa64 %zmm3, 128(%rax) @@ -10946,7 +10945,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vbroadcasti64x4 {{.*#+}} zmm11 = [0,5,4,0,0,6,5,0,0,5,4,0,0,6,5,0] ; AVX512F-SLOW-NEXT: # zmm11 = mem[0,1,2,3,0,1,2,3] ; AVX512F-SLOW-NEXT: vpermd 64(%rax), %zmm11, %zmm0 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm17, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm1, %zmm17, %zmm0 ; AVX512F-SLOW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm0 = ymm6[3,3,3,3,7,7,7,7] ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm1 = ymm3[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15] @@ -10973,7 +10972,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastd 96(%rax), %ymm5 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm5 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm5, %zmm4 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm2, %zmm5, %zmm4 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm5, %zmm15 ; AVX512F-SLOW-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-SLOW-NEXT: vmovdqa 96(%rsi), %xmm2 @@ -11007,7 +11006,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastd 100(%rax), %ymm2 ; AVX512F-SLOW-NEXT: vpbroadcastd 104(%rax), %ymm3 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm16, %zmm2 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm1, %zmm16, %zmm2 ; AVX512F-SLOW-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-SLOW-NEXT: vmovdqa 64(%rcx), %xmm1 ; AVX512F-SLOW-NEXT: vmovdqa 64(%rdx), %xmm2 @@ -11049,7 +11048,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastd 68(%rax), %ymm5 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm9 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm9, %zmm4 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm1, %zmm9, %zmm4 ; AVX512F-SLOW-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-SLOW-NEXT: vmovdqa (%rcx), %xmm1 ; AVX512F-SLOW-NEXT: vmovdqa (%rdx), %xmm5 @@ -11085,7 +11084,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastd (%rax), %ymm2 ; AVX512F-SLOW-NEXT: vpbroadcastd 4(%rax), %ymm3 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm9, %zmm2 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm1, %zmm9, %zmm2 ; AVX512F-SLOW-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-SLOW-NEXT: vmovdqa64 %ymm23, %ymm7 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} ymm1 = ymm7[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15] @@ -11120,7 +11119,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm6[2],ymm1[3,4],ymm6[5],ymm1[6,7,8,9],ymm6[10],ymm1[11,12],ymm6[13],ymm1[14,15] ; AVX512F-SLOW-NEXT: vmovdqa64 %ymm1, %ymm21 ; AVX512F-SLOW-NEXT: vpermd (%rax), %zmm11, %zmm1 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm17, %zmm1 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm2, %zmm17, %zmm1 ; AVX512F-SLOW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm1 = ymm10[3,3,3,3,7,7,7,7] ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm2 = ymm8[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15] @@ -11145,7 +11144,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm11 = ymm11[2,1,3,2] ; AVX512F-SLOW-NEXT: vpbroadcastd 32(%rax), %ymm12 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm25 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm15, %zmm25 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm1, %zmm15, %zmm25 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %xmm1 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rsi), %xmm12 ; AVX512F-SLOW-NEXT: vprold $16, %xmm12, %xmm15 @@ -11178,7 +11177,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastd 36(%rax), %ymm0 ; AVX512F-SLOW-NEXT: vpbroadcastd 40(%rax), %ymm6 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm6, %zmm0, %zmm28 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm28 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm28 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rcx), %xmm6 ; AVX512F-SLOW-NEXT: vmovdqa64 %xmm29, %xmm0 ; AVX512F-SLOW-NEXT: vpshufb %xmm0, %xmm6, %xmm2 @@ -11284,9 +11283,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[4,5,6,7] ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm1, %zmm23 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm0, %zmm1, %zmm23 ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload -; AVX512F-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm29 # 64-byte Folded Reload +; AVX512F-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm29 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535] ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload ; AVX512F-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm17 # 64-byte Folded Reload @@ -11643,7 +11642,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vmovdqa 32(%rax), %ymm0 ; AVX512F-FAST-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-FAST-NEXT: vpermi2d %zmm0, %zmm19, %zmm2 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm3, %zmm22, %zmm2 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm3, %zmm22, %zmm2 ; AVX512F-FAST-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload ; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm13, %ymm0 @@ -11679,7 +11678,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vbroadcasti64x4 {{.*#+}} zmm2 = [0,13,4,0,0,14,5,0,0,13,4,0,0,14,5,0] ; AVX512F-FAST-NEXT: # zmm2 = mem[0,1,2,3,0,1,2,3] ; AVX512F-FAST-NEXT: vpermd %zmm0, %zmm2, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm3, %zmm25, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm3, %zmm25, %zmm0 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm0 ; AVX512F-FAST-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-FAST-NEXT: vpshufb %ymm11, %ymm1, %ymm0 @@ -11776,7 +11775,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastd 68(%rax), %ymm6 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm23 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm2, %zmm4, %zmm23 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm2, %zmm4, %zmm23 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535] ; AVX512F-FAST-NEXT: vpternlogq $184, %zmm1, %zmm2, %zmm23 ; AVX512F-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload @@ -11790,7 +11789,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastd (%rax), %ymm5 ; AVX512F-FAST-NEXT: vpbroadcastd 4(%rax), %ymm6 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm20 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm1, %zmm4, %zmm20 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm1, %zmm4, %zmm20 ; AVX512F-FAST-NEXT: vpternlogq $184, %zmm9, %zmm2, %zmm20 ; AVX512F-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm4[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u] @@ -11835,7 +11834,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [0,13,4,0,0,14,5,0,0,13,4,0,0,14,5,0] ; AVX512F-FAST-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3] ; AVX512F-FAST-NEXT: vpermd %zmm1, %zmm5, %zmm19 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm2, %zmm25, %zmm19 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm2, %zmm25, %zmm19 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm19 ; AVX512F-FAST-NEXT: vmovdqa 32(%rcx), %xmm12 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdx), %xmm8 @@ -11851,7 +11850,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastd 96(%rax), %ymm2 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm21 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm30 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm13, %zmm30, %zmm21 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm13, %zmm30, %zmm21 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm21 ; AVX512F-FAST-NEXT: vmovdqa64 %xmm24, %xmm2 ; AVX512F-FAST-NEXT: vmovdqa64 %xmm26, %xmm1 @@ -11904,7 +11903,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastd 104(%rax), %ymm6 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm6, %zmm2, %zmm2 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm22 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm3, %zmm22, %zmm2 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm3, %zmm22, %zmm2 ; AVX512F-FAST-NEXT: vmovdqa64 %ymm18, %ymm3 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31] ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm6 = ymm16[3,3,3,3,7,7,7,7] @@ -11918,7 +11917,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpermd %ymm14, %ymm31, %ymm9 ; AVX512F-FAST-NEXT: vpbroadcastd 32(%rax), %ymm18 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm18, %zmm9, %zmm9 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm10, %zmm30, %zmm9 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm10, %zmm30, %zmm9 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm9 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0] ; AVX512F-FAST-NEXT: vpternlogq $184, %zmm0, %zmm1, %zmm2 @@ -11944,7 +11943,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastd 36(%rax), %ymm3 ; AVX512F-FAST-NEXT: vpbroadcastd 40(%rax), %ymm5 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm5, %zmm3, %zmm3 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm4, %zmm22, %zmm3 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm4, %zmm22, %zmm3 ; AVX512F-FAST-NEXT: vpternlogq $184, %zmm0, %zmm1, %zmm3 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535] ; AVX512F-FAST-NEXT: vpternlogq $184, %ymm7, %ymm0, %ymm29 @@ -11955,7 +11954,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm26, %zmm0, %zmm1 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm29[0,1,2,3],zmm1[4,5,6,7] ; AVX512F-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm12 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm12 ; AVX512F-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload ; AVX512F-FAST-NEXT: vpunpckhwd (%rsp), %xmm1, %xmm1 # 16-byte Folded Reload ; AVX512F-FAST-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7] diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll index 8ccb4ef56d9c744a517baed1d64ab856b8413ace..f727622682cf237d916175819a377d8f796adecc 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll @@ -567,8 +567,8 @@ define void @store_i8_stride3_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm3 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] ; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3] -; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm2 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] -; AVX512BW-NEXT: # zmm2 = mem[0,1,2,3,0,1,2,3] +; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm2 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] +; AVX512BW-NEXT: # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3] ; AVX512BW-NEXT: vpshufb %ymm2, %ymm0, %ymm0 ; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1 ; AVX512BW-NEXT: vpshufb %zmm2, %zmm1, %zmm1 @@ -1086,8 +1086,8 @@ define void @store_i8_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] ; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3] ; AVX512BW-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm2 -; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] -; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3] +; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] +; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3] ; AVX512BW-NEXT: vpshufb %zmm3, %zmm2, %zmm2 ; AVX512BW-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm4 ; AVX512BW-NEXT: vpshufb %zmm3, %zmm4, %zmm4 diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll index f4fda97c0817a8d638760dd021646c516e6a5ef1..4174fde817472a9892cb2d32407505f85b6d6e25 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll @@ -927,7 +927,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vporq %zmm7, %zmm5, %zmm5 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm6 = ; AVX512F-SLOW-NEXT: vpermd %zmm1, %zmm6, %zmm6 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 ; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u],zero,zero,xmm3[10,11,u],zero,zero,xmm3[12,13,u],zero,zero,xmm3[14,15,u] ; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] @@ -966,7 +966,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm6 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm7 = ; AVX512F-FAST-NEXT: vpermd %zmm6, %zmm7, %zmm6 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 ; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u],zero,zero,xmm3[10,11,u],zero,zero,xmm3[12,13,u],zero,zero,xmm3[14,15,u] ; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] @@ -1886,7 +1886,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vmovdqa (%r8), %xmm6 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm8 = ; AVX512F-SLOW-NEXT: vpermd %zmm6, %zmm8, %zmm6 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm5 = ymm4[u,u,u],zero,ymm4[13,u,u,u],zero,ymm4[14,u,u,u],zero,ymm4[15,u,u,u],zero,ymm4[16,u,u,u],zero,ymm4[17,u,u,u],zero,ymm4[18,u,u] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm8 = ymm3[u,u,u,13],zero,ymm3[u,u,u,14],zero,ymm3[u,u,u,15],zero,ymm3[u,u,u,16],zero,ymm3[u,u,u,17],zero,ymm3[u,u,u,18],zero,ymm3[u,u] ; AVX512F-SLOW-NEXT: vpor %ymm5, %ymm8, %ymm5 @@ -1968,7 +1968,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = mem[0,1,2,3,0,1,2,3] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm7 = ; AVX512F-FAST-NEXT: vpermd %zmm4, %zmm7, %zmm7 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm7 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm7 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm5 = ymm3[u,u,u],zero,ymm3[13,u,u,u],zero,ymm3[14,u,u,u],zero,ymm3[15,u,u,u],zero,ymm3[16,u,u,u],zero,ymm3[17,u,u,u],zero,ymm3[18,u,u] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm8 = ymm2[u,u,u,13],zero,ymm2[u,u,u,14],zero,ymm2[u,u,u,15],zero,ymm2[u,u,u,16],zero,ymm2[u,u,u,17],zero,ymm2[u,u,u,18],zero,ymm2[u,u] ; AVX512F-FAST-NEXT: vpor %ymm5, %ymm8, %ymm5 @@ -3890,36 +3890,37 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm15 = [128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128,128,128] ; AVX512F-SLOW-NEXT: vpshufb %ymm15, %ymm3, %ymm0 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %ymm2 -; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm7 = <12,13,128,15,12,13,14,128,12,13,14,15,128,u,u,u,16,128,18,19,16,17,128,19,16,17,18,128,16,17,18,19> -; AVX512F-SLOW-NEXT: vpshufb %ymm7, %ymm2, %ymm1 +; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm9 = <12,13,128,15,12,13,14,128,12,13,14,15,128,u,u,u,16,128,18,19,16,17,128,19,16,17,18,128,16,17,18,19> +; AVX512F-SLOW-NEXT: vpshufb %ymm9, %ymm2, %ymm1 ; AVX512F-SLOW-NEXT: vpor %ymm0, %ymm1, %ymm0 ; AVX512F-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm8 = <8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11> -; AVX512F-SLOW-NEXT: vpshufb %xmm8, %xmm1, %xmm0 -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm1, %xmm30 +; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm5 = <8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11> +; AVX512F-SLOW-NEXT: vpshufb %xmm5, %xmm1, %xmm0 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm1, %xmm16 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rsi), %xmm4 ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm14 = <128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128> ; AVX512F-SLOW-NEXT: vpshufb %xmm14, %xmm4, %xmm1 ; AVX512F-SLOW-NEXT: vmovdqa64 %xmm4, %xmm31 ; AVX512F-SLOW-NEXT: vpor %xmm0, %xmm1, %xmm0 ; AVX512F-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX512F-SLOW-NEXT: vmovdqa 32(%rcx), %ymm9 +; AVX512F-SLOW-NEXT: vmovdqa 32(%rcx), %ymm8 ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm0 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128] -; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm9, %ymm4 +; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm8, %ymm4 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rdx), %ymm11 ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm1 = ; AVX512F-SLOW-NEXT: vpshufb %ymm1, %ymm11, %ymm10 -; AVX512F-SLOW-NEXT: vporq %ymm4, %ymm10, %ymm18 +; AVX512F-SLOW-NEXT: vpor %ymm4, %ymm10, %ymm4 +; AVX512F-SLOW-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX512F-SLOW-NEXT: vmovdqa 32(%rcx), %xmm13 -; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm5 = <128,6,128,8,u,128,7,128,9,128,11,u,128,10,128,12> -; AVX512F-SLOW-NEXT: vpshufb %xmm5, %xmm13, %xmm4 -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm5, %xmm25 +; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm6 = <128,6,128,8,u,128,7,128,9,128,11,u,128,10,128,12> +; AVX512F-SLOW-NEXT: vpshufb %xmm6, %xmm13, %xmm4 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm6, %xmm25 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rdx), %xmm10 -; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm5 = <6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128> -; AVX512F-SLOW-NEXT: vpshufb %xmm5, %xmm10, %xmm12 -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm5, %xmm26 -; AVX512F-SLOW-NEXT: vporq %xmm4, %xmm12, %xmm19 +; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm6 = <6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128> +; AVX512F-SLOW-NEXT: vpshufb %xmm6, %xmm10, %xmm12 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm6, %xmm26 +; AVX512F-SLOW-NEXT: vporq %xmm4, %xmm12, %xmm20 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm4 = ymm3[11,u,u,10,u,12,u,u,u,u,13,u,15,u,u,14,27,u,u,26,u,28,u,u,u,u,29,u,31,u,u,30] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[3,u,5,u,u,4,u,6,u,8,u,u,7,u,9,u,19,u,21,u,u,20,u,22,u,24,u,u,23,u,25,u] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm22 @@ -3927,31 +3928,31 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[21],zero,zero,ymm2[20],zero,ymm2[22],zero,ymm2[24],zero,zero,ymm2[23],zero,ymm2[25],zero,zero ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm23 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm11[27],zero,zero,ymm11[26],zero,ymm11[28],zero,ymm11[30],zero,zero,ymm11[29],zero,ymm11[31],zero,zero -; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm3 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm9[19],zero,ymm9[21],zero,zero,ymm9[20],zero,ymm9[22],zero,ymm9[24],zero,zero,ymm9[23],zero +; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm3 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm8[19],zero,ymm8[21],zero,zero,ymm8[20],zero,ymm8[22],zero,ymm8[24],zero,zero,ymm8[23],zero ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm24 ; AVX512F-SLOW-NEXT: vmovdqa (%rcx), %ymm12 ; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm12, %ymm0 ; AVX512F-SLOW-NEXT: vmovdqa (%rdx), %ymm6 ; AVX512F-SLOW-NEXT: vpshufb %ymm1, %ymm6, %ymm1 -; AVX512F-SLOW-NEXT: vporq %ymm0, %ymm1, %ymm20 -; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %ymm5 -; AVX512F-SLOW-NEXT: vpshufb %ymm15, %ymm5, %ymm2 +; AVX512F-SLOW-NEXT: vporq %ymm0, %ymm1, %ymm19 +; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %ymm7 +; AVX512F-SLOW-NEXT: vpshufb %ymm15, %ymm7, %ymm2 ; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %ymm4 -; AVX512F-SLOW-NEXT: vpshufb %ymm7, %ymm4, %ymm3 +; AVX512F-SLOW-NEXT: vpshufb %ymm9, %ymm4, %ymm3 ; AVX512F-SLOW-NEXT: vporq %ymm2, %ymm3, %ymm21 ; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %xmm0 -; AVX512F-SLOW-NEXT: vpshufb %xmm8, %xmm0, %xmm2 -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm0, %xmm16 -; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %xmm7 -; AVX512F-SLOW-NEXT: vpshufb %xmm14, %xmm7, %xmm3 -; AVX512F-SLOW-NEXT: vporq %xmm2, %xmm3, %xmm27 +; AVX512F-SLOW-NEXT: vpshufb %xmm5, %xmm0, %xmm3 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm0, %xmm17 +; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %xmm5 +; AVX512F-SLOW-NEXT: vpshufb %xmm14, %xmm5, %xmm9 +; AVX512F-SLOW-NEXT: vporq %xmm3, %xmm9, %xmm27 ; AVX512F-SLOW-NEXT: vmovdqa (%rcx), %xmm1 ; AVX512F-SLOW-NEXT: vmovdqa64 %xmm25, %xmm0 ; AVX512F-SLOW-NEXT: vpshufb %xmm0, %xmm1, %xmm0 -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm1, %xmm17 -; AVX512F-SLOW-NEXT: vmovdqa (%rdx), %xmm8 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm1, %xmm18 +; AVX512F-SLOW-NEXT: vmovdqa (%rdx), %xmm9 ; AVX512F-SLOW-NEXT: vmovdqa64 %xmm26, %xmm1 -; AVX512F-SLOW-NEXT: vpshufb %xmm1, %xmm8, %xmm15 +; AVX512F-SLOW-NEXT: vpshufb %xmm1, %xmm9, %xmm15 ; AVX512F-SLOW-NEXT: vporq %xmm0, %xmm15, %xmm29 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = mem[1,1,2,2] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,1,1] @@ -3965,10 +3966,11 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufb %ymm1, %ymm0, %ymm1 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,2,1,1,4,6,5,5] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,3,2] -; AVX512F-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 +; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm30 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0] +; AVX512F-SLOW-NEXT: vpandnq %ymm0, %ymm30, %ymm0 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm25 ; AVX512F-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm0 = [9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12] -; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm9, %ymm9 +; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm8, %ymm8 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm14 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm12[19],zero,ymm12[21],zero,zero,ymm12[20],zero,ymm12[22],zero,ymm12[24],zero,zero,ymm12[23],zero ; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm12, %ymm12 ; AVX512F-SLOW-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25] @@ -3977,24 +3979,25 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm6, %ymm2 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm6[27],zero,zero,ymm6[26],zero,ymm6[28],zero,ymm6[30],zero,zero,ymm6[29],zero,ymm6[31],zero,zero ; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm10 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3],xmm13[4],xmm10[4],xmm13[5],xmm10[5],xmm13[6],xmm10[6],xmm13[7],xmm10[7] -; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm13 = ymm5[3,u,5,u,u,4,u,6,u,8,u,u,7,u,9,u,19,u,21,u,u,20,u,22,u,24,u,u,23,u,25,u] -; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[11,u,u,10,u,12,u,u,u,u,13,u,15,u,u,14,27,u,u,26,u,28,u,u,u,u,29,u,31,u,u,30] +; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm13 = ymm7[3,u,5,u,u,4,u,6,u,8,u,u,7,u,9,u,19,u,21,u,u,20,u,22,u,24,u,u,23,u,25,u] +; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm7 = ymm7[11,u,u,10,u,12,u,u,u,u,13,u,15,u,u,14,27,u,u,26,u,28,u,u,u,u,29,u,31,u,u,30] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm4[21],zero,zero,ymm4[20],zero,ymm4[22],zero,ymm4[24],zero,zero,ymm4[23],zero,ymm4[25],zero,zero ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm4[26],zero,ymm4[28],zero,zero,ymm4[27],zero,ymm4[29],zero,ymm4[31],zero,zero,ymm4[30],zero -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm30, %xmm1 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm16, %xmm1 ; AVX512F-SLOW-NEXT: vmovdqa64 %xmm31, %xmm4 ; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm4 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm1 = <4,u,5,5,5,5,u,6,6,6,6,u,7,7,7,7> -; AVX512F-SLOW-NEXT: vpermd %zmm15, %zmm1, %zmm30 -; AVX512F-SLOW-NEXT: vmovdqa64 (%r8), %zmm31 +; AVX512F-SLOW-NEXT: vpermd %zmm15, %zmm1, %zmm31 +; AVX512F-SLOW-NEXT: vmovdqa64 (%r8), %zmm16 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm1 = <6,6,6,u,7,7,7,7,u,16,16,16,16,u,17,17> -; AVX512F-SLOW-NEXT: vpermi2d %zmm15, %zmm31, %zmm1 -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm16, %xmm15 -; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm15[0],xmm7[0],xmm15[1],xmm7[1],xmm15[2],xmm7[2],xmm15[3],xmm7[3],xmm15[4],xmm7[4],xmm15[5],xmm7[5],xmm15[6],xmm7[6],xmm15[7],xmm7[7] +; AVX512F-SLOW-NEXT: vpermi2d %zmm15, %zmm16, %zmm1 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm17, %xmm15 +; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm5 = xmm15[0],xmm5[0],xmm15[1],xmm5[1],xmm15[2],xmm5[2],xmm15[3],xmm5[3],xmm15[4],xmm5[4],xmm15[5],xmm5[5],xmm15[6],xmm5[6],xmm15[7],xmm5[7] ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm15 = <0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13> ; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm4, %xmm4 -; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm7, %xmm7 -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm9 = ymm9[2,2,3,3] +; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm5, %xmm5 +; AVX512F-SLOW-NEXT: vinserti32x4 $2, %xmm27, %zmm5, %zmm5 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm11 = ymm11[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,3,3] @@ -4002,65 +4005,64 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm12 = ymm12[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,2,3,3] -; AVX512F-SLOW-NEXT: vinserti32x4 $2, %xmm27, %zmm7, %zmm27 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm18, %xmm15 +; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3],xmm15[4],xmm9[4],xmm15[5],xmm9[5],xmm15[6],xmm9[6],xmm15[7],xmm9[7] ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm15 = <2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8> ; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm10, %xmm10 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm10 = ymm10[0,0,1,1] -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,2,3,3] +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,0,1,1] -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm17, %xmm7 -; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm8 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3],xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7] -; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm8, %xmm8 -; AVX512F-SLOW-NEXT: vinserti32x4 $2, %xmm29, %zmm8, %zmm8 +; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm9, %xmm9 +; AVX512F-SLOW-NEXT: vinserti32x4 $2, %xmm29, %zmm9, %zmm9 ; AVX512F-SLOW-NEXT: vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload ; AVX512F-SLOW-NEXT: # ymm15 = mem[0,0,1,1] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm15, %zmm15 # 32-byte Folded Reload -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm16 = ymm19[0,0,1,1] -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm18, %zmm16, %zmm16 -; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm17 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0] -; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm15, %zmm17, %zmm16 -; AVX512F-SLOW-NEXT: vpternlogq $248, %zmm28, %zmm16, %zmm26 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm17 = ymm20[0,0,1,1] +; AVX512F-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm17 # 32-byte Folded Reload +; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm18 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0] +; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm15, %zmm18, %zmm17 +; AVX512F-SLOW-NEXT: vpternlogq $248, %zmm28, %zmm17, %zmm26 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm15 = zmm22[2,2,3,3,6,6,7,7] +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm17 = zmm23[2,2,3,3,6,6,7,7] +; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm20 = [18374966859431608575,18374966859431608575,18446463693966278400,18446463693966278400,72056498804555775,72056498804555775,18374967950370078975,18374967950370078975] +; AVX512F-SLOW-NEXT: vpternlogq $248, %zmm20, %zmm15, %zmm17 +; AVX512F-SLOW-NEXT: vpandq %ymm20, %ymm8, %ymm8 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm8, %zmm11, %zmm8 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm11 = zmm24[2,2,3,3,6,6,7,7] +; AVX512F-SLOW-NEXT: vporq %zmm11, %zmm8, %zmm8 +; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm11 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255] +; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm17, %zmm11, %zmm8 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm8, %zmm30, %zmm31 ; AVX512F-SLOW-NEXT: vpor %ymm2, %ymm14, %ymm2 -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm20, %zmm2 -; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm14 = [18374966859431608575,18374966859431608575,18446463693966278400,18446463693966278400,72056498804555775,72056498804555775,18374967950370078975,18374967950370078975] -; AVX512F-SLOW-NEXT: vpternlogq $248, %ymm14, %ymm13, %ymm0 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm19, %zmm2 +; AVX512F-SLOW-NEXT: vpternlogq $248, %ymm20, %ymm13, %ymm0 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm21, %zmm0 -; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm2, %zmm17, %zmm0 -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm2 = zmm22[2,2,3,3,6,6,7,7] -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm13 = zmm23[2,2,3,3,6,6,7,7] -; AVX512F-SLOW-NEXT: vpternlogq $248, %zmm14, %zmm2, %zmm13 -; AVX512F-SLOW-NEXT: vpternlogq $248, %ymm14, %ymm12, %ymm6 -; AVX512F-SLOW-NEXT: vpand %ymm14, %ymm9, %ymm2 -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm11, %zmm2 -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm9 = zmm24[2,2,3,3,6,6,7,7] -; AVX512F-SLOW-NEXT: vporq %zmm9, %zmm2, %zmm2 -; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm9 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255] -; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm13, %zmm9, %zmm2 -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm6, %zmm6 -; AVX512F-SLOW-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm3 +; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm2, %zmm18, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogq $248, %ymm20, %ymm12, %ymm6 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm6, %zmm2 +; AVX512F-SLOW-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm3 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm3 -; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm6, %zmm9, %zmm3 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm30 +; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm2, %zmm11, %zmm3 ; AVX512F-SLOW-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm25 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm1 -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm0 = zmm27[0,0,1,1,4,4,5,5] -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm2 = zmm8[0,0,1,1,4,4,5,5] +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm1 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm0 = zmm5[0,0,1,1,4,4,5,5] +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm2 = zmm9[0,0,1,1,4,4,5,5] ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm0 = -; AVX512F-SLOW-NEXT: vpermd %zmm31, %zmm0, %zmm0 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 +; AVX512F-SLOW-NEXT: vpermd %zmm16, %zmm0, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm25, 64(%r9) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm0, (%r9) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm1, 128(%r9) -; AVX512F-SLOW-NEXT: vmovdqa64 %zmm30, 256(%r9) +; AVX512F-SLOW-NEXT: vmovdqa64 %zmm31, 256(%r9) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm26, 192(%r9) ; AVX512F-SLOW-NEXT: vzeroupper ; AVX512F-SLOW-NEXT: retq ; ; AVX512F-FAST-LABEL: store_i8_stride5_vf64: ; AVX512F-FAST: # %bb.0: -; AVX512F-FAST-NEXT: pushq %rax +; AVX512F-FAST-NEXT: subq $24, %rsp ; AVX512F-FAST-NEXT: vmovdqa 32(%rsi), %ymm3 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm8 = [128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128,128,128] ; AVX512F-FAST-NEXT: vpshufb %ymm8, %ymm3, %ymm0 @@ -4069,14 +4071,14 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb %ymm11, %ymm1, %ymm2 ; AVX512F-FAST-NEXT: vpor %ymm0, %ymm2, %ymm0 ; AVX512F-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %xmm2 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm6 = <8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11> -; AVX512F-FAST-NEXT: vpshufb %xmm6, %xmm2, %xmm0 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm2, %xmm19 +; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %xmm0 +; AVX512F-FAST-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm5 = <8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11> +; AVX512F-FAST-NEXT: vpshufb %xmm5, %xmm0, %xmm0 ; AVX512F-FAST-NEXT: vmovdqa 32(%rsi), %xmm4 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm13 = <128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128> ; AVX512F-FAST-NEXT: vpshufb %xmm13, %xmm4, %xmm2 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm4, %xmm18 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm4, %xmm19 ; AVX512F-FAST-NEXT: vpor %xmm0, %xmm2, %xmm0 ; AVX512F-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX512F-FAST-NEXT: vmovdqa 32(%rcx), %ymm7 @@ -4092,9 +4094,9 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb %xmm4, %xmm12, %xmm2 ; AVX512F-FAST-NEXT: vmovdqa64 %xmm4, %xmm25 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdx), %xmm10 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm5 = <6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128> -; AVX512F-FAST-NEXT: vpshufb %xmm5, %xmm10, %xmm4 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm5, %xmm26 +; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm6 = <6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128> +; AVX512F-FAST-NEXT: vpshufb %xmm6, %xmm10, %xmm4 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm6, %xmm26 ; AVX512F-FAST-NEXT: vpor %xmm2, %xmm4, %xmm2 ; AVX512F-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm2 = ymm3[11,u,u,10,u,12,u,u,u,u,13,u,15,u,u,14,27,u,u,26,u,28,u,u,u,u,29,u,31,u,u,30] @@ -4104,140 +4106,139 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[21],zero,zero,ymm1[20],zero,ymm1[22],zero,ymm1[24],zero,zero,ymm1[23],zero,ymm1[25],zero,zero ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm22 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm9[27],zero,zero,ymm9[26],zero,ymm9[28],zero,ymm9[30],zero,zero,ymm9[29],zero,ymm9[31],zero,zero -; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128] -; AVX512F-FAST-NEXT: # ymm3 = mem[0,1,0,1] -; AVX512F-FAST-NEXT: vpshufb %ymm3, %ymm7, %ymm2 -; AVX512F-FAST-NEXT: vmovdqa64 %ymm3, %ymm31 +; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm2 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm7[19],zero,ymm7[21],zero,zero,ymm7[20],zero,ymm7[22],zero,ymm7[24],zero,zero,ymm7[23],zero ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm23 -; AVX512F-FAST-NEXT: vmovdqa (%rcx), %ymm5 -; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm5, %ymm0 +; AVX512F-FAST-NEXT: vmovdqa (%rcx), %ymm4 +; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm4, %ymm0 ; AVX512F-FAST-NEXT: vmovdqa (%rdx), %ymm14 ; AVX512F-FAST-NEXT: vpshufb %ymm15, %ymm14, %ymm1 ; AVX512F-FAST-NEXT: vporq %ymm0, %ymm1, %ymm24 ; AVX512F-FAST-NEXT: vmovdqa (%rsi), %ymm15 ; AVX512F-FAST-NEXT: vpshufb %ymm8, %ymm15, %ymm0 -; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm4 -; AVX512F-FAST-NEXT: vpshufb %ymm11, %ymm4, %ymm1 +; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm6 +; AVX512F-FAST-NEXT: vpshufb %ymm11, %ymm6, %ymm1 ; AVX512F-FAST-NEXT: vporq %ymm0, %ymm1, %ymm20 -; AVX512F-FAST-NEXT: vmovdqa (%rdi), %xmm1 -; AVX512F-FAST-NEXT: vpshufb %xmm6, %xmm1, %xmm0 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm1, %xmm16 +; AVX512F-FAST-NEXT: vmovdqa (%rdi), %xmm0 +; AVX512F-FAST-NEXT: vpshufb %xmm5, %xmm0, %xmm1 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm0, %xmm16 ; AVX512F-FAST-NEXT: vmovdqa (%rsi), %xmm3 ; AVX512F-FAST-NEXT: vpshufb %xmm13, %xmm3, %xmm2 -; AVX512F-FAST-NEXT: vporq %xmm0, %xmm2, %xmm28 +; AVX512F-FAST-NEXT: vporq %xmm1, %xmm2, %xmm28 ; AVX512F-FAST-NEXT: vmovdqa (%rcx), %xmm1 ; AVX512F-FAST-NEXT: vmovdqa64 %xmm25, %xmm0 -; AVX512F-FAST-NEXT: vpshufb %xmm0, %xmm1, %xmm0 +; AVX512F-FAST-NEXT: vpshufb %xmm0, %xmm1, %xmm8 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm1, %xmm18 +; AVX512F-FAST-NEXT: vmovdqa (%rdx), %xmm1 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm26, %xmm0 +; AVX512F-FAST-NEXT: vpshufb %xmm0, %xmm1, %xmm11 ; AVX512F-FAST-NEXT: vmovdqa64 %xmm1, %xmm17 -; AVX512F-FAST-NEXT: vmovdqa (%rdx), %xmm11 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm26, %xmm1 -; AVX512F-FAST-NEXT: vpshufb %xmm1, %xmm11, %xmm8 -; AVX512F-FAST-NEXT: vporq %xmm0, %xmm8, %xmm29 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm0 = [1,1,2,2,2,2,2,2] +; AVX512F-FAST-NEXT: vporq %xmm8, %xmm11, %xmm29 +; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm11 = [1,1,2,2,2,2,2,2] ; AVX512F-FAST-NEXT: vmovdqa 32(%r8), %ymm8 -; AVX512F-FAST-NEXT: vpermd %ymm8, %ymm0, %ymm0 +; AVX512F-FAST-NEXT: vpermd %ymm8, %ymm11, %ymm11 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm25 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255] -; AVX512F-FAST-NEXT: vpandnq %ymm0, %ymm25, %ymm0 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm1 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128] -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm8, %ymm13 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm13, %zmm0, %zmm26 -; AVX512F-FAST-NEXT: vmovdqa (%r8), %ymm2 -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm2, %ymm1 -; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm30 = <4,u,5,5,5,5,u,6,30,30,30,u,31,31,31,31> -; AVX512F-FAST-NEXT: vpermd %ymm2, %ymm30, %ymm27 -; AVX512F-FAST-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm27, %ymm27 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm27, %zmm1, %zmm27 -; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm1 = [9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12] -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm7, %ymm7 -; AVX512F-FAST-NEXT: vmovdqa64 %ymm31, %ymm0 -; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm5, %ymm13 -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm5, %ymm6 -; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25] -; AVX512F-FAST-NEXT: # ymm5 = mem[0,1,0,1] -; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm9, %ymm9 -; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm14, %ymm5 +; AVX512F-FAST-NEXT: vpandnq %ymm11, %ymm25, %ymm11 +; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm0 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128] +; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm8, %ymm13 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm13, %zmm11, %zmm26 +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm31 = <4,u,5,5,5,5,u,6,30,30,30,u,31,31,31,31> +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm30 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0] +; AVX512F-FAST-NEXT: vmovdqa (%r8), %ymm11 +; AVX512F-FAST-NEXT: vpermd %ymm11, %ymm31, %ymm27 +; AVX512F-FAST-NEXT: vpandnq %ymm27, %ymm30, %ymm27 +; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm11, %ymm0 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm27, %zmm0, %zmm27 +; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm0 = [9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12] +; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm7, %ymm7 +; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm13 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm4[19],zero,ymm4[21],zero,zero,ymm4[20],zero,ymm4[22],zero,ymm4[24],zero,zero,ymm4[23],zero +; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm4, %ymm5 +; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25] +; AVX512F-FAST-NEXT: # ymm0 = mem[0,1,0,1] +; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm9, %ymm9 +; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm14, %ymm4 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm14[27],zero,zero,ymm14[26],zero,ymm14[28],zero,ymm14[30],zero,zero,ymm14[29],zero,ymm14[31],zero,zero ; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3],xmm12[4],xmm10[4],xmm12[5],xmm10[5],xmm12[6],xmm10[6],xmm12[7],xmm10[7] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm12 = ymm15[3,u,5,u,u,4,u,6,u,8,u,u,7,u,9,u,19,u,21,u,u,20,u,22,u,24,u,u,23,u,25,u] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm15 = ymm15[11,u,u,10,u,12,u,u,u,u,13,u,15,u,u,14,27,u,u,26,u,28,u,u,u,u,29,u,31,u,u,30] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm31 = ymm7[2,2,3,3] -; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm4[21],zero,zero,ymm4[20],zero,ymm4[22],zero,ymm4[24],zero,zero,ymm4[23],zero,ymm4[25],zero,zero -; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm4[26],zero,ymm4[28],zero,zero,ymm4[27],zero,ymm4[29],zero,ymm4[31],zero,zero,ymm4[30],zero -; AVX512F-FAST-NEXT: vmovdqa64 %xmm19, %xmm0 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm18, %xmm1 +; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm2 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm6[21],zero,zero,ymm6[20],zero,ymm6[22],zero,ymm6[24],zero,zero,ymm6[23],zero,ymm6[25],zero,zero +; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm6[26],zero,ymm6[28],zero,zero,ymm6[27],zero,ymm6[29],zero,ymm6[31],zero,zero,ymm6[30],zero +; AVX512F-FAST-NEXT: vmovdqa64 %xmm19, %xmm1 +; AVX512F-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] ; AVX512F-FAST-NEXT: vmovdqa64 %xmm16, %xmm1 ; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = <0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13> ; AVX512F-FAST-NEXT: vpshufb %xmm1, %xmm0, %xmm0 ; AVX512F-FAST-NEXT: vpshufb %xmm1, %xmm3, %xmm1 -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm3 = ymm9[2,2,3,3] +; AVX512F-FAST-NEXT: vinserti32x4 $2, %xmm28, %zmm1, %zmm28 +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm1 = ymm7[2,2,3,3] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm7 = ymm9[2,2,3,3] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm9 = ymm13[2,2,3,3] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,2,3,3] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,3,3] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm12 = ymm12[2,2,3,3] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,2,3,3] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm13 = ymm14[2,2,3,3] -; AVX512F-FAST-NEXT: vinserti32x4 $2, %xmm28, %zmm1, %zmm18 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm14 = <2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8> -; AVX512F-FAST-NEXT: vpshufb %xmm14, %xmm10, %xmm10 +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,3,3] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm13 = ymm5[2,2,3,3] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,2,3,3] +; AVX512F-FAST-NEXT: vmovdqa64 %xmm18, %xmm3 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm17, %xmm5 +; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} xmm5 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7] +; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm3 = <2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8> +; AVX512F-FAST-NEXT: vpshufb %xmm3, %xmm10, %xmm10 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm10 = ymm10[0,0,1,1] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,3,3] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,3,3] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,2,3,3] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,1,1] -; AVX512F-FAST-NEXT: vmovdqa64 %xmm17, %xmm1 -; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} xmm11 = xmm1[0],xmm11[0],xmm1[1],xmm11[1],xmm1[2],xmm11[2],xmm1[3],xmm11[3],xmm1[4],xmm11[4],xmm1[5],xmm11[5],xmm1[6],xmm11[6],xmm1[7],xmm11[7] -; AVX512F-FAST-NEXT: vpshufb %xmm14, %xmm11, %xmm11 -; AVX512F-FAST-NEXT: vinserti32x4 $2, %xmm29, %zmm11, %zmm11 -; AVX512F-FAST-NEXT: vmovdqa64 (%r8), %zmm14 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm14, %zmm2 +; AVX512F-FAST-NEXT: vpshufb %xmm3, %xmm5, %xmm3 +; AVX512F-FAST-NEXT: vinserti32x4 $2, %xmm29, %zmm3, %zmm3 +; AVX512F-FAST-NEXT: vmovdqa64 (%r8), %zmm5 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm11, %zmm5, %zmm11 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm21 = zmm21[2,2,3,3,6,6,7,7] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm22 = zmm22[2,2,3,3,6,6,7,7] -; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm28 = [18374966859431608575,18374966859431608575,18446463693966278400,18446463693966278400,72056498804555775,72056498804555775,18374967950370078975,18374967950370078975] -; AVX512F-FAST-NEXT: vpternlogq $248, %zmm28, %zmm21, %zmm22 -; AVX512F-FAST-NEXT: vpandq %ymm28, %ymm31, %ymm21 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm21, %zmm3, %zmm3 -; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm21 = zmm23[2,2,3,3,6,6,7,7] -; AVX512F-FAST-NEXT: vporq %zmm21, %zmm3, %zmm3 -; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm21 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255] -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm22, %zmm21, %zmm3 -; AVX512F-FAST-NEXT: vpermt2d %zmm14, %zmm30, %zmm8 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm8 +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm29 = [18374966859431608575,18374966859431608575,18446463693966278400,18446463693966278400,72056498804555775,72056498804555775,18374967950370078975,18374967950370078975] +; AVX512F-FAST-NEXT: vpternlogq $248, %zmm29, %zmm21, %zmm22 +; AVX512F-FAST-NEXT: vpandq %ymm29, %ymm1, %ymm1 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm7, %zmm1 +; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm7 = zmm23[2,2,3,3,6,6,7,7] +; AVX512F-FAST-NEXT: vporq %zmm7, %zmm1, %zmm1 +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm7 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255] +; AVX512F-FAST-NEXT: vpternlogq $226, %zmm22, %zmm7, %zmm1 +; AVX512F-FAST-NEXT: vpermt2d %zmm5, %zmm31, %zmm8 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm1, %zmm30, %zmm8 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm8, 256(%r9) -; AVX512F-FAST-NEXT: vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload -; AVX512F-FAST-NEXT: # ymm3 = mem[0,0,1,1] -; AVX512F-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 32-byte Folded Reload +; AVX512F-FAST-NEXT: vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload +; AVX512F-FAST-NEXT: # ymm1 = mem[0,0,1,1] +; AVX512F-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload ; AVX512F-FAST-NEXT: vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload ; AVX512F-FAST-NEXT: # ymm8 = mem[0,0,1,1] ; AVX512F-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 32-byte Folded Reload ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm16 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0] -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm3, %zmm16, %zmm8 +; AVX512F-FAST-NEXT: vpternlogq $226, %zmm1, %zmm16, %zmm8 ; AVX512F-FAST-NEXT: vpternlogq $248, %zmm25, %zmm8, %zmm26 -; AVX512F-FAST-NEXT: vpor %ymm5, %ymm9, %ymm3 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm24, %zmm3 -; AVX512F-FAST-NEXT: vpternlogq $248, %ymm28, %ymm12, %ymm7 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm20, %zmm5 -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm3, %zmm16, %zmm5 -; AVX512F-FAST-NEXT: vpternlogq $248, %ymm28, %ymm6, %ymm13 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm10, %zmm13, %zmm3 -; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm15, %ymm4 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm4, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm3, %zmm21, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm27 -; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm3 = <6,6,6,u,7,7,7,7,u,8,8,8,8,u,9,9> -; AVX512F-FAST-NEXT: vpermd %zmm14, %zmm3, %zmm3 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm3 -; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm0 = zmm18[0,0,1,1,4,4,5,5] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm11[0,0,1,1,4,4,5,5] -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 +; AVX512F-FAST-NEXT: vpor %ymm4, %ymm9, %ymm1 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm24, %zmm1 +; AVX512F-FAST-NEXT: vpternlogq $248, %ymm29, %ymm12, %ymm2 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm20, %zmm2 +; AVX512F-FAST-NEXT: vpternlogq $226, %zmm1, %zmm16, %zmm2 +; AVX512F-FAST-NEXT: vpternlogq $248, %ymm29, %ymm13, %ymm14 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm10, %zmm14, %zmm1 +; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm15, %ymm6 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm6, %zmm0 +; AVX512F-FAST-NEXT: vpternlogq $226, %zmm1, %zmm7, %zmm0 +; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm27 +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm1 = <6,6,6,u,7,7,7,7,u,8,8,8,8,u,9,9> +; AVX512F-FAST-NEXT: vpermd %zmm5, %zmm1, %zmm1 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 +; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm0 = zmm28[0,0,1,1,4,4,5,5] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm2 = zmm3[0,0,1,1,4,4,5,5] +; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm0 = -; AVX512F-FAST-NEXT: vpermd %zmm2, %zmm0, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0 +; AVX512F-FAST-NEXT: vpermd %zmm11, %zmm0, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm27, 64(%r9) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm0, (%r9) -; AVX512F-FAST-NEXT: vmovdqa64 %zmm3, 128(%r9) +; AVX512F-FAST-NEXT: vmovdqa64 %zmm1, 128(%r9) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm26, 192(%r9) -; AVX512F-FAST-NEXT: popq %rax +; AVX512F-FAST-NEXT: addq $24, %rsp ; AVX512F-FAST-NEXT: vzeroupper ; AVX512F-FAST-NEXT: retq ; @@ -4424,37 +4425,37 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FAST-LABEL: store_i8_stride5_vf64: ; AVX512BW-FAST: # %bb.0: ; AVX512BW-FAST-NEXT: vmovdqa64 (%r8), %zmm5 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdx), %ymm21 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdx), %ymm20 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} ymm1 = [128,128,12,13,128,128,128,128,14,128,128,128,14,15,128,128,128,128,16,128,128,128,16,17,128,128,128,128,18,128,128,128] -; AVX512BW-FAST-NEXT: vpshufb %ymm1, %ymm21, %ymm0 +; AVX512BW-FAST-NEXT: vpshufb %ymm1, %ymm20, %ymm0 ; AVX512BW-FAST-NEXT: vmovdqa 32(%rcx), %ymm13 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128] ; AVX512BW-FAST-NEXT: vpshufb %ymm2, %ymm13, %ymm3 ; AVX512BW-FAST-NEXT: vpor %ymm0, %ymm3, %ymm0 ; AVX512BW-FAST-NEXT: vmovdqa (%rcx), %xmm6 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rcx), %xmm18 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rcx), %xmm17 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm8 = <128,6,128,8,u,128,7,128,9,128,11,u,128,10,128,12> -; AVX512BW-FAST-NEXT: vpshufb %xmm8, %xmm18, %xmm3 +; AVX512BW-FAST-NEXT: vpshufb %xmm8, %xmm17, %xmm3 ; AVX512BW-FAST-NEXT: vmovdqa (%rdx), %xmm7 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdx), %xmm20 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdx), %xmm19 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm10 = <6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128> -; AVX512BW-FAST-NEXT: vpshufb %xmm10, %xmm20, %xmm4 +; AVX512BW-FAST-NEXT: vpshufb %xmm10, %xmm19, %xmm4 ; AVX512BW-FAST-NEXT: vpor %xmm3, %xmm4, %xmm3 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,1,1] ; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0 ; AVX512BW-FAST-NEXT: vmovdqa (%rdi), %xmm9 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %xmm17 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %xmm16 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm12 = <8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11> -; AVX512BW-FAST-NEXT: vpshufb %xmm12, %xmm17, %xmm3 +; AVX512BW-FAST-NEXT: vpshufb %xmm12, %xmm16, %xmm3 ; AVX512BW-FAST-NEXT: vmovdqa (%rsi), %xmm11 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rsi), %xmm19 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rsi), %xmm18 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm14 = <128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128> -; AVX512BW-FAST-NEXT: vpshufb %xmm14, %xmm19, %xmm4 +; AVX512BW-FAST-NEXT: vpshufb %xmm14, %xmm18, %xmm4 ; AVX512BW-FAST-NEXT: vpor %xmm3, %xmm4, %xmm3 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm15 = ymm3[0,0,1,1] -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %ymm16 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %ymm21 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} ymm3 = <3,3,3,u,4,4,4,4> -; AVX512BW-FAST-NEXT: vpermd %ymm16, %ymm3, %ymm22 +; AVX512BW-FAST-NEXT: vpermd %ymm21, %ymm3, %ymm22 ; AVX512BW-FAST-NEXT: vmovdqa64 32(%rsi), %ymm23 ; AVX512BW-FAST-NEXT: vpbroadcastq {{.*#+}} ymm4 = [0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14] ; AVX512BW-FAST-NEXT: movl $138547332, %eax # imm = 0x8421084 @@ -4464,80 +4465,77 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FAST-NEXT: movabsq $-8330787646191410408, %rax # imm = 0x8C6318C6318C6318 ; AVX512BW-FAST-NEXT: kmovq %rax, %k2 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm15, %zmm0 {%k2} -; AVX512BW-FAST-NEXT: vmovdqa64 32(%r8), %ymm24 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%r8), %ymm22 ; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm15 = <1,1,2,2,2,2,2,2,27,27,27,27,u,28,28,28> -; AVX512BW-FAST-NEXT: vpermi2d %zmm5, %zmm24, %zmm15 +; AVX512BW-FAST-NEXT: vpermi2d %zmm5, %zmm22, %zmm15 ; AVX512BW-FAST-NEXT: movabsq $4760450083537948804, %rax # imm = 0x4210842108421084 ; AVX512BW-FAST-NEXT: kmovq %rax, %k3 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm15, %zmm0 {%k3} -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm22 = zmm23[0,1,2,3],mem[4,5,6,7] +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm23 = zmm23[0,1,2,3],mem[4,5,6,7] ; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm15 = -; AVX512BW-FAST-NEXT: vpshufb %zmm15, %zmm22, %zmm22 -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm22 = zmm22[2,2,3,3,6,6,7,7] -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm23 = zmm16[0,1,2,3],mem[4,5,6,7] -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm16 = -; AVX512BW-FAST-NEXT: vpshufb %zmm16, %zmm23, %zmm23 +; AVX512BW-FAST-NEXT: vpshufb %zmm15, %zmm23, %zmm23 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm23 = zmm23[2,2,3,3,6,6,7,7] -; AVX512BW-FAST-NEXT: vporq %zmm22, %zmm23, %zmm23 -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm22 = zmm21[0,1,2,3],mem[4,5,6,7] -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm21 = -; AVX512BW-FAST-NEXT: vpshufb %zmm21, %zmm22, %zmm22 -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm25 = zmm22[2,2,3,3,6,6,7,7] +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm21[0,1,2,3],mem[4,5,6,7] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm21 = zmm21[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm21[21],zero,zero,zmm21[20],zero,zmm21[22],zero,zmm21[24],zero,zero,zmm21[23],zero,zmm21[25],zero,zero,zmm21[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm21[58],zero,zmm21[60],zero,zero,zmm21[59],zero,zmm21[61],zero,zmm21[63],zero,zero,zmm21[62],zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm21 = zmm21[2,2,3,3,6,6,7,7] +; AVX512BW-FAST-NEXT: vporq %zmm23, %zmm21, %zmm21 +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm20[0,1,2,3],mem[4,5,6,7] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm20 = zmm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19],zero,zmm20[21],zero,zmm20[21,20],zero,zmm20[22],zero,zmm20[24],zero,zmm20[22,23],zero,zmm20[25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,57],zero,zmm20[59],zero,zero,zmm20[58],zero,zmm20[60],zero,zmm20[62],zero,zero,zmm20[61],zero,zmm20[63],zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm20 = zmm20[2,2,3,3,6,6,7,7] ; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm13[0,1,2,3],mem[4,5,6,7] -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm22 = -; AVX512BW-FAST-NEXT: vpshufb %zmm22, %zmm13, %zmm13 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm13 = zmm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm13[19],zero,zmm13[21],zero,zero,zmm13[20],zero,zmm13[22],zero,zmm13[24],zero,zero,zmm13[23],zero,zmm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm13[59],zero,zero,zmm13[58],zero,zmm13[60],zero,zmm13[62],zero,zero,zmm13[61],zero,zmm13[63],zero,zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm13 = zmm13[2,2,3,3,6,6,7,7] -; AVX512BW-FAST-NEXT: vporq %zmm25, %zmm13, %zmm13 +; AVX512BW-FAST-NEXT: vporq %zmm20, %zmm13, %zmm13 ; AVX512BW-FAST-NEXT: movabsq $1785168781326730801, %rax # imm = 0x18C6318C6318C631 ; AVX512BW-FAST-NEXT: kmovq %rax, %k3 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm23, %zmm13 {%k3} -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm23 = [4,6,5,5,5,5,4,6,30,30,30,30,31,31,31,31] -; AVX512BW-FAST-NEXT: vpermi2d %zmm5, %zmm24, %zmm23 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm21, %zmm13 {%k3} +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm20 = [4,6,5,5,5,5,4,6,30,30,30,30,31,31,31,31] +; AVX512BW-FAST-NEXT: vpermi2d %zmm5, %zmm22, %zmm20 ; AVX512BW-FAST-NEXT: movabsq $-8925843906633654008, %rax # imm = 0x8421084210842108 ; AVX512BW-FAST-NEXT: kmovq %rax, %k4 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm23, %zmm13 {%k4} -; AVX512BW-FAST-NEXT: vmovdqa64 (%rcx), %ymm23 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm24 = ymm23[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm23[27],zero,zero,ymm23[26],zero,ymm23[28],zero,ymm23[30],zero,zero,ymm23[29],zero,ymm23[31],zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm25 = ymm24[2,2,3,3] -; AVX512BW-FAST-NEXT: vmovdqa64 (%rdx), %ymm24 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm26 = ymm24[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm24[27],zero,zero,ymm24[26],zero,ymm24[28],zero,ymm24[30],zero,zero,ymm24[29],zero,ymm24[31],zero,zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm26 = ymm26[2,2,3,3] -; AVX512BW-FAST-NEXT: vporq %ymm25, %ymm26, %ymm25 -; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm18 = xmm18[0],xmm20[0],xmm18[1],xmm20[1],xmm18[2],xmm20[2],xmm18[3],xmm20[3],xmm18[4],xmm20[4],xmm18[5],xmm20[5],xmm18[6],xmm20[6],xmm18[7],xmm20[7] -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} xmm26 = <2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8> -; AVX512BW-FAST-NEXT: vpshufb %xmm26, %xmm18, %xmm18 -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm18 = ymm18[0,0,1,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm18, %zmm25, %zmm25 -; AVX512BW-FAST-NEXT: vmovdqa64 (%rsi), %ymm18 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm20 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,ymm18[26],zero,ymm18[28],zero,zero,zero,zero,ymm18[29],zero,ymm18[31],zero,zero,ymm18[30] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm27 = ymm20[2,2,3,3] -; AVX512BW-FAST-NEXT: vmovdqa64 (%rdi), %ymm20 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm28 = ymm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm20[26],zero,ymm20[28],zero,zero,ymm20[27],zero,ymm20[29],zero,ymm20[31],zero,zero,ymm20[30],zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm28 = ymm28[2,2,3,3] -; AVX512BW-FAST-NEXT: vporq %ymm27, %ymm28, %ymm27 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm20, %zmm13 {%k4} +; AVX512BW-FAST-NEXT: vmovdqa64 (%rcx), %ymm20 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm21 = ymm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm20[27],zero,zero,ymm20[26],zero,ymm20[28],zero,ymm20[30],zero,zero,ymm20[29],zero,ymm20[31],zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm21 = ymm21[2,2,3,3] +; AVX512BW-FAST-NEXT: vmovdqa64 (%rdx), %ymm22 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm23 = ymm22[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm22[27],zero,zero,ymm22[26],zero,ymm22[28],zero,ymm22[30],zero,zero,ymm22[29],zero,ymm22[31],zero,zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm23 = ymm23[2,2,3,3] +; AVX512BW-FAST-NEXT: vporq %ymm21, %ymm23, %ymm21 ; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm17 = xmm17[0],xmm19[0],xmm17[1],xmm19[1],xmm17[2],xmm19[2],xmm17[3],xmm19[3],xmm17[4],xmm19[4],xmm17[5],xmm19[5],xmm17[6],xmm19[6],xmm17[7],xmm19[7] -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} xmm19 = <0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13> +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} xmm19 = <2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8> ; AVX512BW-FAST-NEXT: vpshufb %xmm19, %xmm17, %xmm17 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm17 = ymm17[0,0,1,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm17, %zmm27, %zmm17 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm25, %zmm17 {%k3} -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm25 = [6,6,6,6,7,7,7,7,8,8,8,8,8,8,9,9] -; AVX512BW-FAST-NEXT: vpermd %zmm5, %zmm25, %zmm5 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm17, %zmm21, %zmm21 +; AVX512BW-FAST-NEXT: vmovdqa64 (%rsi), %ymm17 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm23 = ymm17[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,ymm17[26],zero,ymm17[28],zero,zero,zero,zero,ymm17[29],zero,ymm17[31],zero,zero,ymm17[30] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm23 = ymm23[2,2,3,3] +; AVX512BW-FAST-NEXT: vmovdqa64 (%rdi), %ymm24 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm25 = ymm24[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm24[26],zero,ymm24[28],zero,zero,ymm24[27],zero,ymm24[29],zero,ymm24[31],zero,zero,ymm24[30],zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm25 = ymm25[2,2,3,3] +; AVX512BW-FAST-NEXT: vporq %ymm23, %ymm25, %ymm23 +; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm16 = xmm16[0],xmm18[0],xmm16[1],xmm18[1],xmm16[2],xmm18[2],xmm16[3],xmm18[3],xmm16[4],xmm18[4],xmm16[5],xmm18[5],xmm16[6],xmm18[6],xmm16[7],xmm18[7] +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} xmm18 = <0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13> +; AVX512BW-FAST-NEXT: vpshufb %xmm18, %xmm16, %xmm16 +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm16 = ymm16[0,0,1,1] +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm16, %zmm23, %zmm16 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm21, %zmm16 {%k3} +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm21 = [6,6,6,6,7,7,7,7,8,8,8,8,8,8,9,9] +; AVX512BW-FAST-NEXT: vpermd %zmm5, %zmm21, %zmm5 ; AVX512BW-FAST-NEXT: movabsq $2380225041768974402, %rax # imm = 0x2108421084210842 ; AVX512BW-FAST-NEXT: kmovq %rax, %k3 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm5, %zmm17 {%k3} +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm5, %zmm16 {%k3} ; AVX512BW-FAST-NEXT: vpshufb %xmm8, %xmm6, %xmm5 ; AVX512BW-FAST-NEXT: vpshufb %xmm10, %xmm7, %xmm8 ; AVX512BW-FAST-NEXT: vpor %xmm5, %xmm8, %xmm5 ; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7] -; AVX512BW-FAST-NEXT: vpshufb %xmm26, %xmm6, %xmm6 +; AVX512BW-FAST-NEXT: vpshufb %xmm19, %xmm6, %xmm6 ; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm5, %zmm6, %zmm5 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm5 = zmm5[0,0,1,1,4,4,5,5] ; AVX512BW-FAST-NEXT: vpshufb %xmm12, %xmm9, %xmm6 ; AVX512BW-FAST-NEXT: vpshufb %xmm14, %xmm11, %xmm7 ; AVX512BW-FAST-NEXT: vpor %xmm6, %xmm7, %xmm6 ; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm9[0],xmm11[0],xmm9[1],xmm11[1],xmm9[2],xmm11[2],xmm9[3],xmm11[3],xmm9[4],xmm11[4],xmm9[5],xmm11[5],xmm9[6],xmm11[6],xmm9[7],xmm11[7] -; AVX512BW-FAST-NEXT: vpshufb %xmm19, %xmm7, %xmm7 +; AVX512BW-FAST-NEXT: vpshufb %xmm18, %xmm7, %xmm7 ; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm6, %zmm7, %zmm6 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm6 = zmm6[0,0,1,1,4,4,5,5] ; AVX512BW-FAST-NEXT: movabsq $-4165393823095705204, %rax # imm = 0xC6318C6318C6318C @@ -4549,22 +4547,22 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FAST-NEXT: movabsq $595056260442243600, %rax # imm = 0x842108421084210 ; AVX512BW-FAST-NEXT: kmovq %rax, %k3 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm7, %zmm6 {%k3} -; AVX512BW-FAST-NEXT: vpshufb %ymm21, %ymm24, %ymm7 -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3] -; AVX512BW-FAST-NEXT: vpshufb %ymm22, %ymm23, %ymm8 -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm8 = ymm8[2,2,3,3] -; AVX512BW-FAST-NEXT: vpor %ymm7, %ymm8, %ymm7 -; AVX512BW-FAST-NEXT: vpshufb %ymm1, %ymm24, %ymm1 -; AVX512BW-FAST-NEXT: vpshufb %ymm2, %ymm23, %ymm2 +; AVX512BW-FAST-NEXT: vpshufb %ymm1, %ymm22, %ymm1 +; AVX512BW-FAST-NEXT: vpshufb %ymm2, %ymm20, %ymm2 ; AVX512BW-FAST-NEXT: vpor %ymm1, %ymm2, %ymm1 -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm1, %zmm1 -; AVX512BW-FAST-NEXT: vpshufb %ymm15, %ymm18, %ymm2 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm22[18,19],zero,ymm22[21],zero,ymm22[21,20],zero,ymm22[22],zero,ymm22[24],zero,ymm22[22,23],zero,ymm22[25] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,3,3] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm7 = ymm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm20[19],zero,ymm20[21],zero,zero,ymm20[20],zero,ymm20[22],zero,ymm20[24],zero,zero,ymm20[23],zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3] +; AVX512BW-FAST-NEXT: vpor %ymm2, %ymm7, %ymm2 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1 +; AVX512BW-FAST-NEXT: vpshufb %ymm15, %ymm17, %ymm2 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,3,3] -; AVX512BW-FAST-NEXT: vpshufb %ymm16, %ymm20, %ymm7 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm24[21],zero,zero,ymm24[20],zero,ymm24[22],zero,ymm24[24],zero,zero,ymm24[23],zero,ymm24[25],zero,zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3] ; AVX512BW-FAST-NEXT: vpor %ymm2, %ymm7, %ymm2 -; AVX512BW-FAST-NEXT: vpermd %ymm20, %ymm3, %ymm3 -; AVX512BW-FAST-NEXT: vpshufb %ymm4, %ymm18, %ymm3 {%k1} +; AVX512BW-FAST-NEXT: vpermd %ymm24, %ymm3, %ymm3 +; AVX512BW-FAST-NEXT: vpshufb %ymm4, %ymm17, %ymm3 {%k1} ; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm2 {%k2} ; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm1 = <3,3,3,3,u,4,4,4,12,14,13,13,13,13,12,14> @@ -4574,7 +4572,7 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm2 {%k1} ; AVX512BW-FAST-NEXT: vmovdqa64 %zmm2, 64(%r9) ; AVX512BW-FAST-NEXT: vmovdqa64 %zmm6, (%r9) -; AVX512BW-FAST-NEXT: vmovdqa64 %zmm17, 128(%r9) +; AVX512BW-FAST-NEXT: vmovdqa64 %zmm16, 128(%r9) ; AVX512BW-FAST-NEXT: vmovdqa64 %zmm13, 256(%r9) ; AVX512BW-FAST-NEXT: vmovdqa64 %zmm0, 192(%r9) ; AVX512BW-FAST-NEXT: vzeroupper diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll index 3bc7b6e958223d14e8a8780124c734ae3f3c57bb..b1493847cdb8520fc44f0220c1292a45cc8a33e2 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll @@ -1724,10 +1724,10 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-LABEL: store_i8_stride6_vf32: ; AVX512F-SLOW: # %bb.0: ; AVX512F-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %ymm2 -; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %ymm3 -; AVX512F-SLOW-NEXT: vmovdqa (%rdx), %ymm4 -; AVX512F-SLOW-NEXT: vmovdqa (%rcx), %ymm5 +; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %ymm3 +; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %ymm5 +; AVX512F-SLOW-NEXT: vmovdqa (%rdx), %ymm2 +; AVX512F-SLOW-NEXT: vmovdqa (%rcx), %ymm4 ; AVX512F-SLOW-NEXT: vmovdqa (%r8), %ymm0 ; AVX512F-SLOW-NEXT: vmovdqa (%r9), %ymm1 ; AVX512F-SLOW-NEXT: vmovdqa (%rcx), %xmm7 @@ -1735,7 +1735,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[10,11,8,9,6,7,12,13,14,15,14,15,14,15,14,15] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,0,0,1] -; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm9 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[16],ymm5[16],ymm4[17],ymm5[17],ymm4[18],ymm5[18],ymm4[19],ymm5[19],ymm4[20],ymm5[20],ymm4[21],ymm5[21],ymm4[22],ymm5[22],ymm4[23],ymm5[23] +; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm9 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[16],ymm4[16],ymm2[17],ymm4[17],ymm2[18],ymm4[18],ymm2[19],ymm4[19],ymm2[20],ymm4[20],ymm2[21],ymm4[21],ymm2[22],ymm4[22],ymm2[23],ymm4[23] ; AVX512F-SLOW-NEXT: vprold $16, %ymm9, %ymm9 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm6, %zmm6 @@ -1744,13 +1744,13 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm11 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[8,9,6,7,12,13,10,11,14,15,14,15,14,15,14,15] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm11 = ymm11[0,0,0,1] -; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm12 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[16],ymm3[16],ymm2[17],ymm3[17],ymm2[18],ymm3[18],ymm2[19],ymm3[19],ymm2[20],ymm3[20],ymm2[21],ymm3[21],ymm2[22],ymm3[22],ymm2[23],ymm3[23] +; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm12 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[16],ymm5[16],ymm3[17],ymm5[17],ymm3[18],ymm5[18],ymm3[19],ymm5[19],ymm3[20],ymm5[20],ymm3[21],ymm5[21],ymm3[22],ymm5[22],ymm3[23],ymm5[23] ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} ymm12 = ymm12[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15] ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm13 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm14 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535] -; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm6, %zmm14, %zmm13 +; AVX512F-SLOW-NEXT: vpternlogd $226, %zmm6, %zmm14, %zmm13 ; AVX512F-SLOW-NEXT: vmovdqa (%r9), %xmm11 ; AVX512F-SLOW-NEXT: vmovdqa (%r8), %xmm12 ; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15] @@ -1761,7 +1761,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm15 = ymm15[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,3] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm15, %zmm6, %zmm6 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm13, %zmm6 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm13, %zmm6 ; AVX512F-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm13 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0] ; AVX512F-SLOW-NEXT: vpshufb %xmm13, %xmm9, %xmm15 ; AVX512F-SLOW-NEXT: vpshufb %xmm13, %xmm10, %xmm13 @@ -1769,17 +1769,17 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm15 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7] ; AVX512F-SLOW-NEXT: vprold $16, %xmm15, %xmm15 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm13, %zmm15, %zmm13 -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm15 = zmm13[0,0,0,1,4,4,4,5] -; AVX512F-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm13 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10] -; AVX512F-SLOW-NEXT: vpshufb %xmm13, %xmm7, %xmm7 -; AVX512F-SLOW-NEXT: vpshufb %xmm13, %xmm8, %xmm8 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm13 = zmm13[0,0,0,1,4,4,4,5] +; AVX512F-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm15 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10] +; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm7, %xmm7 +; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm8, %xmm8 ; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7] ; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm8 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3],xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7] ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,3,2,1,4,5,6,7] ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,6,5] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm8 = zmm7[0,0,0,1,4,4,4,5] -; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm8 +; AVX512F-SLOW-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm13, %zmm8 ; AVX512F-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm7 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0] ; AVX512F-SLOW-NEXT: vpshufb %xmm7, %xmm11, %xmm9 ; AVX512F-SLOW-NEXT: vpshufb %xmm7, %xmm12, %xmm10 @@ -1789,33 +1789,31 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,4,4,4] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm10, %zmm9 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm9 = zmm9[0,0,0,1,4,4,4,5] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm8, %zmm14, %zmm9 -; AVX512F-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm8 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0] -; AVX512F-SLOW-NEXT: vpshufb %ymm8, %ymm3, %ymm10 -; AVX512F-SLOW-NEXT: vpshufb %ymm8, %ymm2, %ymm8 -; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm8 = ymm8[0],ymm10[0],ymm8[1],ymm10[1],ymm8[2],ymm10[2],ymm8[3],ymm10[3],ymm8[4],ymm10[4],ymm8[5],ymm10[5],ymm8[6],ymm10[6],ymm8[7],ymm10[7],ymm8[16],ymm10[16],ymm8[17],ymm10[17],ymm8[18],ymm10[18],ymm8[19],ymm10[19],ymm8[20],ymm10[20],ymm8[21],ymm10[21],ymm8[22],ymm10[22],ymm8[23],ymm10[23] -; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} ymm10 = ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15],ymm4[24],ymm5[24],ymm4[25],ymm5[25],ymm4[26],ymm5[26],ymm4[27],ymm5[27],ymm4[28],ymm5[28],ymm4[29],ymm5[29],ymm4[30],ymm5[30],ymm4[31],ymm5[31] -; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31] +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm8, %zmm14, %zmm9 +; AVX512F-SLOW-NEXT: vpshufb %ymm15, %ymm4, %ymm8 +; AVX512F-SLOW-NEXT: vpshufb %ymm15, %ymm2, %ymm10 +; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm8 = ymm10[0],ymm8[0],ymm10[1],ymm8[1],ymm10[2],ymm8[2],ymm10[3],ymm8[3],ymm10[4],ymm8[4],ymm10[5],ymm8[5],ymm10[6],ymm8[6],ymm10[7],ymm8[7],ymm10[16],ymm8[16],ymm10[17],ymm8[17],ymm10[18],ymm8[18],ymm10[19],ymm8[19],ymm10[20],ymm8[20],ymm10[21],ymm8[21],ymm10[22],ymm8[22],ymm10[23],ymm8[23] +; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} ymm10 = ymm3[8],ymm5[8],ymm3[9],ymm5[9],ymm3[10],ymm5[10],ymm3[11],ymm5[11],ymm3[12],ymm5[12],ymm3[13],ymm5[13],ymm3[14],ymm5[14],ymm3[15],ymm5[15],ymm3[24],ymm5[24],ymm3[25],ymm5[25],ymm3[26],ymm5[26],ymm3[27],ymm5[27],ymm3[28],ymm5[28],ymm3[29],ymm5[29],ymm3[30],ymm5[30],ymm3[31],ymm5[31] +; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm8, %zmm8 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm8 = zmm8[2,2,2,3,6,6,6,7] -; AVX512F-SLOW-NEXT: vpshufb %ymm13, %ymm5, %ymm5 -; AVX512F-SLOW-NEXT: vpshufb %ymm13, %ymm4, %ymm4 -; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[16],ymm5[16],ymm4[17],ymm5[17],ymm4[18],ymm5[18],ymm4[19],ymm5[19],ymm4[20],ymm5[20],ymm4[21],ymm5[21],ymm4[22],ymm5[22],ymm4[23],ymm5[23] -; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31] -; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31] -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm4, %zmm2 +; AVX512F-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm10 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0] +; AVX512F-SLOW-NEXT: vpshufb %ymm10, %ymm5, %ymm5 +; AVX512F-SLOW-NEXT: vpshufb %ymm10, %ymm3, %ymm3 +; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[16],ymm5[16],ymm3[17],ymm5[17],ymm3[18],ymm5[18],ymm3[19],ymm5[19],ymm3[20],ymm5[20],ymm3[21],ymm5[21],ymm3[22],ymm5[22],ymm3[23],ymm5[23] +; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15],ymm2[24],ymm4[24],ymm2[25],ymm4[25],ymm2[26],ymm4[26],ymm2[27],ymm4[27],ymm2[28],ymm4[28],ymm2[29],ymm4[29],ymm2[30],ymm4[30],ymm2[31],ymm4[31] +; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31] +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm2 = zmm2[2,2,2,3,6,6,6,7] -; AVX512F-SLOW-NEXT: vbroadcasti64x4 {{.*#+}} zmm3 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535] -; AVX512F-SLOW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3] -; AVX512F-SLOW-NEXT: vpternlogq $202, %zmm8, %zmm2, %zmm3 -; AVX512F-SLOW-NEXT: vpshufb %ymm7, %ymm1, %ymm2 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm2 +; AVX512F-SLOW-NEXT: vpshufb %ymm7, %ymm1, %ymm3 ; AVX512F-SLOW-NEXT: vpshufb %ymm7, %ymm0, %ymm4 -; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[4],ymm2[4],ymm4[5],ymm2[5],ymm4[6],ymm2[6],ymm4[7],ymm2[7],ymm4[16],ymm2[16],ymm4[17],ymm2[17],ymm4[18],ymm2[18],ymm4[19],ymm2[19],ymm4[20],ymm2[20],ymm4[21],ymm2[21],ymm4[22],ymm2[22],ymm4[23],ymm2[23] +; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[16],ymm3[16],ymm4[17],ymm3[17],ymm4[18],ymm3[18],ymm4[19],ymm3[19],ymm4[20],ymm3[20],ymm4[21],ymm3[21],ymm4[22],ymm3[22],ymm4[23],ymm3[23] ; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,26,27,24,25,22,23,28,29,26,27,28,29,30,31] -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm0 = zmm0[2,2,2,3,6,6,6,7] -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm0, 128(%rax) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm9, (%rax) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm6, 64(%rax) @@ -1831,34 +1829,32 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vmovdqa (%rcx), %ymm5 ; AVX512F-FAST-NEXT: vmovdqa (%r8), %ymm0 ; AVX512F-FAST-NEXT: vmovdqa (%r9), %ymm1 -; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm6 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0] -; AVX512F-FAST-NEXT: vpshufb %ymm6, %ymm3, %ymm7 -; AVX512F-FAST-NEXT: vpshufb %ymm6, %ymm2, %ymm6 -; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm6 = ymm6[0],ymm7[0],ymm6[1],ymm7[1],ymm6[2],ymm7[2],ymm6[3],ymm7[3],ymm6[4],ymm7[4],ymm6[5],ymm7[5],ymm6[6],ymm7[6],ymm6[7],ymm7[7],ymm6[16],ymm7[16],ymm6[17],ymm7[17],ymm6[18],ymm7[18],ymm6[19],ymm7[19],ymm6[20],ymm7[20],ymm6[21],ymm7[21],ymm6[22],ymm7[22],ymm6[23],ymm7[23] -; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} ymm7 = ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15],ymm4[24],ymm5[24],ymm4[25],ymm5[25],ymm4[26],ymm5[26],ymm4[27],ymm5[27],ymm4[28],ymm5[28],ymm4[29],ymm5[29],ymm4[30],ymm5[30],ymm4[31],ymm5[31] -; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31] -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6 -; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm6 = zmm6[2,2,2,3,6,6,6,7] ; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm7 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10] -; AVX512F-FAST-NEXT: vpshufb %ymm7, %ymm5, %ymm8 -; AVX512F-FAST-NEXT: vpshufb %ymm7, %ymm4, %ymm9 -; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm8 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[16],ymm8[16],ymm9[17],ymm8[17],ymm9[18],ymm8[18],ymm9[19],ymm8[19],ymm9[20],ymm8[20],ymm9[21],ymm8[21],ymm9[22],ymm8[22],ymm9[23],ymm8[23] -; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} ymm9 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31] -; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31] +; AVX512F-FAST-NEXT: vpshufb %ymm7, %ymm5, %ymm6 +; AVX512F-FAST-NEXT: vpshufb %ymm7, %ymm4, %ymm8 +; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm6 = ymm8[0],ymm6[0],ymm8[1],ymm6[1],ymm8[2],ymm6[2],ymm8[3],ymm6[3],ymm8[4],ymm6[4],ymm8[5],ymm6[5],ymm8[6],ymm6[6],ymm8[7],ymm6[7],ymm8[16],ymm6[16],ymm8[17],ymm6[17],ymm8[18],ymm6[18],ymm8[19],ymm6[19],ymm8[20],ymm6[20],ymm8[21],ymm6[21],ymm8[22],ymm6[22],ymm8[23],ymm6[23] +; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} ymm8 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31] +; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31] +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm8, %zmm6, %zmm6 +; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm6 = zmm6[2,2,2,3,6,6,6,7] +; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm8 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0] +; AVX512F-FAST-NEXT: vpshufb %ymm8, %ymm3, %ymm9 +; AVX512F-FAST-NEXT: vpshufb %ymm8, %ymm2, %ymm8 +; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm8 = ymm8[0],ymm9[0],ymm8[1],ymm9[1],ymm8[2],ymm9[2],ymm8[3],ymm9[3],ymm8[4],ymm9[4],ymm8[5],ymm9[5],ymm8[6],ymm9[6],ymm8[7],ymm9[7],ymm8[16],ymm9[16],ymm8[17],ymm9[17],ymm8[18],ymm9[18],ymm8[19],ymm9[19],ymm8[20],ymm9[20],ymm8[21],ymm9[21],ymm8[22],ymm9[22],ymm8[23],ymm9[23] +; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} ymm9 = ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15],ymm4[24],ymm5[24],ymm4[25],ymm5[25],ymm4[26],ymm5[26],ymm4[27],ymm5[27],ymm4[28],ymm5[28],ymm4[29],ymm5[29],ymm4[30],ymm5[30],ymm4[31],ymm5[31] +; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm9, %zmm8, %zmm8 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm8 = zmm8[2,2,2,3,6,6,6,7] -; AVX512F-FAST-NEXT: vbroadcasti64x4 {{.*#+}} zmm9 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535] -; AVX512F-FAST-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3] -; AVX512F-FAST-NEXT: vpternlogq $202, %zmm6, %zmm8, %zmm9 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm8 ; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm10 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0] ; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm1, %ymm6 -; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm0, %ymm8 -; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm6 = ymm8[0],ymm6[0],ymm8[1],ymm6[1],ymm8[2],ymm6[2],ymm8[3],ymm6[3],ymm8[4],ymm6[4],ymm8[5],ymm6[5],ymm8[6],ymm6[6],ymm8[7],ymm6[7],ymm8[16],ymm6[16],ymm8[17],ymm6[17],ymm8[18],ymm6[18],ymm8[19],ymm6[19],ymm8[20],ymm6[20],ymm8[21],ymm6[21],ymm8[22],ymm6[22],ymm8[23],ymm6[23] -; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} ymm8 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] -; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,26,27,24,25,22,23,28,29,26,27,28,29,30,31] -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm8, %zmm6, %zmm6 +; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm0, %ymm9 +; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm6 = ymm9[0],ymm6[0],ymm9[1],ymm6[1],ymm9[2],ymm6[2],ymm9[3],ymm6[3],ymm9[4],ymm6[4],ymm9[5],ymm6[5],ymm9[6],ymm6[6],ymm9[7],ymm6[7],ymm9[16],ymm6[16],ymm9[17],ymm6[17],ymm9[18],ymm6[18],ymm9[19],ymm6[19],ymm9[20],ymm6[20],ymm9[21],ymm6[21],ymm9[22],ymm6[22],ymm9[23],ymm6[23] +; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} ymm9 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] +; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,26,27,24,25,22,23,28,29,26,27,28,29,30,31] +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm9, %zmm6, %zmm6 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm6 = zmm6[2,2,2,3,6,6,6,7] -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm6 ; AVX512F-FAST-NEXT: vmovdqa (%rcx), %xmm9 ; AVX512F-FAST-NEXT: vpshufb %xmm7, %xmm9, %xmm8 ; AVX512F-FAST-NEXT: vmovdqa (%rdx), %xmm11 @@ -1878,7 +1874,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vprold $16, %xmm14, %xmm14 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm13, %zmm14, %zmm13 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm14 = zmm13[0,0,0,1,4,4,4,5] -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm14 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm14 ; AVX512F-FAST-NEXT: vmovdqa (%r9), %xmm12 ; AVX512F-FAST-NEXT: vpshufb %xmm10, %xmm12, %xmm15 ; AVX512F-FAST-NEXT: vmovdqa (%r8), %xmm13 @@ -1889,7 +1885,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vinserti32x4 $2, %xmm10, %zmm15, %zmm10 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm10 = zmm10[0,0,0,1,4,4,4,5] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm14, %zmm15, %zmm10 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm14, %zmm15, %zmm10 ; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[16],ymm5[16],ymm4[17],ymm5[17],ymm4[18],ymm5[18],ymm4[19],ymm5[19],ymm4[20],ymm5[20],ymm4[21],ymm5[21],ymm4[22],ymm5[22],ymm4[23],ymm5[23] ; AVX512F-FAST-NEXT: vprold $16, %ymm4, %ymm4 ; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm11[8],xmm9[8],xmm11[9],xmm9[9],xmm11[10],xmm9[10],xmm11[11],xmm9[11],xmm11[12],xmm9[12],xmm11[13],xmm9[13],xmm11[14],xmm9[14],xmm11[15],xmm9[15] @@ -1901,13 +1897,13 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[8,9,6,7,12,13,10,11,14,15,14,15,14,15,14,15] ; AVX512F-FAST-NEXT: vpermt2q %zmm2, %zmm9, %zmm3 -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm5, %zmm15, %zmm3 +; AVX512F-FAST-NEXT: vpternlogd $226, %zmm5, %zmm15, %zmm3 ; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,18,19,16,17,22,23,24,25,24,25,24,25,24,25] ; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,8,9,6,7,12,13,10,11,12,13,14,15] ; AVX512F-FAST-NEXT: vpermt2q %zmm0, %zmm9, %zmm1 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm1 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm1 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm1, 64(%rax) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm10, (%rax) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm6, 128(%rax) @@ -4306,7 +4302,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm28[0,1,2,3],zmm1[4,5,6,7] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm13, %zmm1 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm8 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm5, %zmm8, %zmm1 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm5, %zmm8, %zmm1 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm5 = ymm29[0,0,0,1] ; AVX512F-SLOW-NEXT: vprold $16, %ymm30, %ymm9 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm11 = ymm21[0,0,0,1] @@ -4336,7 +4332,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm2 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm15[0,1,2,3],zmm2[4,5,6,7] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm29, %zmm28, %zmm5 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm8, %zmm5 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm2, %zmm8, %zmm5 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm2 = zmm27[0,0,0,1,4,4,4,5] ; AVX512F-SLOW-NEXT: vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm8 = mem[0,0,0,1,4,4,4,5] @@ -4350,7 +4346,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm8 = mem[0,0,0,1,4,4,4,5] ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm10 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm10, %zmm8 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm2, %zmm10, %zmm8 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm2 = zmm26[0,0,0,1,4,4,4,5] ; AVX512F-SLOW-NEXT: vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm11 = mem[0,0,0,1,4,4,4,5] @@ -4362,7 +4358,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm31[0,1,2,3],zmm2[4,5,6,7] ; AVX512F-SLOW-NEXT: vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm11 = mem[0,0,0,1,4,4,4,5] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm10, %zmm11 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm2, %zmm10, %zmm11 ; AVX512F-SLOW-NEXT: vpermq $234, (%rsp), %zmm2 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm2 = mem[2,2,2,3,6,6,6,7] ; AVX512F-SLOW-NEXT: vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Folded Reload @@ -4382,11 +4378,11 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm2 = mem[2,2,2,3,6,6,6,7] ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm6 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm6, %zmm2 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm0, %zmm6, %zmm2 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm0[4,5,6,7] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm3 = zmm23[2,2,2,3,6,6,6,7] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm6, %zmm3 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm0, %zmm6, %zmm3 ; AVX512F-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm3, 128(%rax) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm2, 320(%rax) @@ -4605,13 +4601,13 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm9, %zmm0, %zmm7 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm7 = zmm13[0,1,2,3],zmm7[4,5,6,7] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm8 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm7, %zmm8, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm7, %zmm8, %zmm6 ; AVX512F-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-FAST-NEXT: vmovdqa64 %zmm6, 256(%rax) ; AVX512F-FAST-NEXT: vpternlogq $184, %ymm11, %ymm16, %ymm12 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm12, %zmm0, %zmm6 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm6[4,5,6,7] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm2, %zmm8, %zmm3 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm2, %zmm8, %zmm3 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm3, 64(%rax) ; AVX512F-FAST-NEXT: vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Folded Reload ; AVX512F-FAST-NEXT: # zmm2 = mem[2,2,2,3,6,6,6,7] @@ -4634,13 +4630,13 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Folded Reload ; AVX512F-FAST-NEXT: # zmm7 = mem[2,2,2,3,6,6,6,7] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm8 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm3, %zmm8, %zmm7 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm3, %zmm8, %zmm7 ; AVX512F-FAST-NEXT: vpternlogq $184, %ymm6, %ymm2, %ymm31 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm30, %zmm0, %zmm3 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm3 = zmm31[0,1,2,3],zmm3[4,5,6,7] ; AVX512F-FAST-NEXT: vpermq $234, (%rsp), %zmm6 # 64-byte Folded Reload ; AVX512F-FAST-NEXT: # zmm6 = mem[2,2,2,3,6,6,6,7] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm3, %zmm8, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm3, %zmm8, %zmm6 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm3 = zmm23[0,0,0,1,4,4,4,5] ; AVX512F-FAST-NEXT: vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload ; AVX512F-FAST-NEXT: # zmm8 = mem[0,0,0,1,4,4,4,5] @@ -4658,11 +4654,11 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm5[0,1,2,3],zmm2[4,5,6,7] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm3 = zmm26[0,0,0,1,4,4,4,5] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm4 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm2, %zmm4, %zmm3 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm2, %zmm4, %zmm3 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm28[0,0,0,1,4,4,4,5] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm0, %zmm4, %zmm1 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm0, %zmm4, %zmm1 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm3, 192(%rax) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm6, 128(%rax) diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll index 54ed0f184827a515967c5991ae8b7b50a916a3d5..04c1bd53225206c0565b7604e97a62e2625bdee7 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll @@ -659,7 +659,7 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[4,12],zero,ymm1[u,u,u,u,5,13],zero,ymm1[u,u,u,u,6,14,22,u,u,u,u],zero,zero,ymm1[23,u,u,u,u,u,u,u,u] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1 ; AVX512F-SLOW-NEXT: vporq %zmm0, %zmm1, %zmm0 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-SLOW-NEXT: vextracti32x4 $2, %zmm0, 32(%rax) ; AVX512F-SLOW-NEXT: vextracti32x4 $3, %zmm0, %xmm1 ; AVX512F-SLOW-NEXT: vmovq %xmm1, 48(%rax) @@ -1560,7 +1560,7 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,0,8],zero,ymm0[u,u,u,u,1,9],zero,ymm0[u,u,u,u,18,26],zero,ymm0[u,u,u,u,19,27],zero,ymm0[u,u,u,u] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm7, %zmm0, %zmm0 ; AVX512F-SLOW-NEXT: vporq %zmm1, %zmm0, %zmm0 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-SLOW-NEXT: vmovdqa %xmm5, 96(%rax) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm0, (%rax) ; AVX512F-SLOW-NEXT: vmovdqa %ymm4, 64(%rax) @@ -1631,7 +1631,7 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[0,4],zero,ymm1[u,u,u,u,1,5],zero,ymm1[u,u,u,u,2,6],zero,ymm1[u,u,u,u,19,23],zero,ymm1[u,u,u,u,24,28],zero,ymm1[u] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm6, %zmm1 ; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm1 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm1 ; AVX512F-FAST-NEXT: vmovdqa %xmm3, 96(%rax) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512F-FAST-NEXT: vmovdqa %ymm2, 64(%rax) @@ -3342,7 +3342,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm8, %zmm8 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm8 = zmm8[2,3,2,3,6,7,6,7] ; AVX512F-SLOW-NEXT: vporq %zmm7, %zmm8, %zmm7 -; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm7 +; AVX512F-SLOW-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm7 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm8 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,u,23,u,u,u,u,26,u,24,u,u,u,u,27,u] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[2,3,2,3] ; AVX512F-SLOW-NEXT: vbroadcasti32x4 {{.*#+}} ymm16 = [18374967954648269055,71777218572844800,18374967954648269055,71777218572844800] @@ -3371,7 +3371,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm11, %zmm10, %zmm10 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm10 = zmm10[2,3,2,3,6,7,6,7] ; AVX512F-SLOW-NEXT: vporq %zmm10, %zmm8, %zmm8 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm8 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm8 ; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm8 ; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %xmm11 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm11[u],zero,xmm11[7],zero,xmm11[5,u,u,u],zero,xmm11[8],zero,xmm11[6,u,u,u],zero @@ -3391,7 +3391,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm14 = xmm14[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9] ; AVX512F-SLOW-NEXT: vinserti32x4 $2, %xmm13, %zmm14, %zmm13 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm18 = zmm13[0,1,0,1,4,5,4,5] -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm18 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm18 ; AVX512F-SLOW-NEXT: vmovdqa (%r9), %xmm13 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm10 = zero,xmm13[4,u,u,u],zero,xmm13[7],zero,xmm13[5,u,u,u],zero,xmm13[8],zero,xmm13[6] ; AVX512F-SLOW-NEXT: vmovdqa (%r8), %xmm14 @@ -3407,7 +3407,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm0, %zmm0 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm10 = zmm0[0,0,1,0,4,4,5,4] -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm19, %zmm10 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm19, %zmm10 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm18, %zmm10 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm1[14,u,u],zero,zero,zero,zero,ymm1[15,u,u],zero,zero,zero,zero,ymm1[16,u,u],zero,zero,zero,zero,ymm1[17,u,u],zero,zero,zero,zero,ymm1[18] ; AVX512F-SLOW-NEXT: vmovdqa64 %ymm1, %ymm19 @@ -3426,7 +3426,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm7 = ymm7[0,1,0,1] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm7, %zmm1 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = zero,ymm5[u,u,u,u,u,14],zero,ymm5[u,u,u,u,u,15],zero,ymm5[u,u,u,u,u,16],zero,ymm5[u,u,u,u,u,17],zero,ymm5[u,u,u] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm7 = ymm6[13,u,u,u,u,u],zero,ymm6[14,u,u,u,u,u],zero,ymm6[15,u,u,u,u,u],zero,ymm6[16,u,u,u,u,u],zero,ymm6[17,u,u,u] ; AVX512F-SLOW-NEXT: vpor %ymm0, %ymm7, %ymm0 @@ -3437,8 +3437,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm7 = xmm15[0,1,2,3,4,5,5,6] ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm7 = ymm7[0,1,0,1] -; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm9 = [255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255] -; AVX512F-SLOW-NEXT: vpandn %ymm7, %ymm9, %ymm7 +; AVX512F-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm7 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm9 = zero,ymm4[13,u,u,u,u],zero,zero,ymm4[14,u,u,u,u],zero,zero,ymm4[15,u,u,u,u],zero,zero,ymm4[16,u,u,u,u],zero,zero,ymm4[17,u,u] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm7, %zmm7 ; AVX512F-SLOW-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm7 @@ -3502,7 +3501,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9] ; AVX512F-FAST-NEXT: vinserti32x4 $2, %xmm10, %zmm13, %zmm10 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm15 = zmm10[0,1,0,1,4,5,4,5] -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm15 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm15 ; AVX512F-FAST-NEXT: vmovdqa (%r10), %xmm10 ; AVX512F-FAST-NEXT: vpshuflw {{.*#+}} xmm7 = xmm10[1,1,0,0,4,5,6,7] ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm13 = @@ -3519,7 +3518,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u] ; AVX512F-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm7, %zmm0 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm7 = zmm0[0,1,0,1,4,5,4,5] -; AVX512F-FAST-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm16, %zmm7 +; AVX512F-FAST-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm16, %zmm7 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm7 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm4[14,u,u],zero,zero,zero,zero,ymm4[15,u,u],zero,zero,zero,zero,ymm4[16,u,u],zero,zero,zero,zero,ymm4[17,u,u],zero,zero,zero,zero,ymm4[18] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm15 = ymm2[0,1,14],zero,ymm2[u,u,0,1,14,15],zero,ymm2[u,u,13,2,3,16],zero,ymm2[u,u,28,29,16,17],zero,ymm2[u,u,19,28,29,18],zero @@ -3535,7 +3534,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm11, %zmm8, %zmm9 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm9 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm9 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm0 = zero,ymm1[u,u,u,u,u,14],zero,ymm1[u,u,u,u,u,15],zero,ymm1[u,u,u,u,u,16],zero,ymm1[u,u,u,u,u,17],zero,ymm1[u,u,u] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm8 = ymm3[13,u,u,u,u,u],zero,ymm3[14,u,u,u,u,u],zero,ymm3[15,u,u,u,u,u],zero,ymm3[16,u,u,u,u,u],zero,ymm3[17,u,u,u] ; AVX512F-FAST-NEXT: vpor %ymm0, %ymm8, %ymm0 @@ -3547,8 +3546,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [2,2,3,3,2,2,3,3] ; AVX512F-FAST-NEXT: # ymm10 = mem[0,1,0,1] ; AVX512F-FAST-NEXT: vpermd %ymm8, %ymm10, %ymm8 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm10 = [255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255] -; AVX512F-FAST-NEXT: vpandn %ymm8, %ymm10, %ymm8 +; AVX512F-FAST-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm8 ; AVX512F-FAST-NEXT: vmovdqa64 %ymm17, %ymm13 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm10 = zero,ymm13[13,u,u,u,u],zero,zero,ymm13[14,u,u,u,u],zero,zero,ymm13[15,u,u,u,u],zero,zero,ymm13[16,u,u,u,u],zero,zero,ymm13[17,u,u] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm10, %zmm8, %zmm8 @@ -3576,7 +3574,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm11 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm11 = zmm11[2,3,2,3,6,7,6,7] ; AVX512F-FAST-NEXT: vporq %zmm10, %zmm11, %zmm10 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm10 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm10 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm3[20],zero,ymm3[18],zero,zero,zero,zero,ymm3[21],zero,ymm3[19],zero,zero,zero,zero,ymm3[22] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm11 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm3[25],zero,ymm3[23],zero,zero,zero,zero,ymm3[26],zero,ymm3[24],zero,zero ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm11, %zmm0, %zmm0 @@ -3593,7 +3591,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm12 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm12 = ymm12[2,3,2,3] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm11 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm11 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm11 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,u,u,u,u,30,u,28,u,u,u,u,31,u,29,u] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3] @@ -7623,19 +7621,19 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm3 = ymm14[0,0,1,0] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm13, %zmm3 ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm3 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm3 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm3 ; AVX512F-SLOW-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm4 = mem[2,3,2,3,6,7,6,7] -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm4 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm4 ; AVX512F-SLOW-NEXT: vporq %ymm15, %ymm18, %ymm5 ; AVX512F-SLOW-NEXT: vporq %ymm19, %ymm20, %ymm6 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm5 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm6[0,1,2,3],zmm5[4,5,6,7] -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm16 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm16 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm16 ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload -; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm26 +; AVX512F-SLOW-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm26 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm17, %zmm26 ; AVX512F-SLOW-NEXT: vporq %ymm21, %ymm22, %ymm1 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1 @@ -7648,7 +7646,7 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm1 = zmm12[0,1,0,1,4,5,4,5] ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm0 = zmm7[0,0,1,0,4,4,5,4] -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm0 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0 ; AVX512F-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm0, (%rax) @@ -8039,7 +8037,7 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm4 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25] ; AVX512F-FAST-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Folded Reload ; AVX512F-FAST-NEXT: # zmm22 = mem[2,3,2,3,6,7,6,7] -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm22 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm22 ; AVX512F-FAST-NEXT: vpermq $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload ; AVX512F-FAST-NEXT: # ymm0 = mem[0,1,0,1] ; AVX512F-FAST-NEXT: vpermq $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm23 # 32-byte Folded Reload @@ -8061,11 +8059,11 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm24 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm23, %zmm24 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm19, %zmm2, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm21 +; AVX512F-FAST-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm21 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm21 ; AVX512F-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm0 # 32-byte Folded Reload ; AVX512F-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm0 ; AVX512F-FAST-NEXT: vpor %ymm12, %ymm15, %ymm2 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm2 @@ -8078,14 +8076,14 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm5 = zmm31[0,1,0,1,4,5,4,5] ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm5 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm6, %zmm17, %zmm2 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm18, %zmm2 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm18, %zmm2 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm2 ; AVX512F-FAST-NEXT: vpor %ymm1, %ymm13, %ymm1 ; AVX512F-FAST-NEXT: vpor %ymm11, %ymm14, %ymm5 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm5[0,1,2,3],zmm1[4,5,6,7] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm4, %zmm20, %zmm4 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm4 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm4 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm4 ; AVX512F-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-FAST-NEXT: vmovdqa64 %zmm4, 128(%rax) @@ -8415,250 +8413,253 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; ; AVX512BW-FAST-LABEL: store_i8_stride7_vf64: ; AVX512BW-FAST: # %bb.0: -; AVX512BW-FAST-NEXT: subq $200, %rsp +; AVX512BW-FAST-NEXT: subq $72, %rsp ; AVX512BW-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX512BW-FAST-NEXT: vmovdqa64 (%rdi), %zmm9 -; AVX512BW-FAST-NEXT: vmovdqu64 %zmm9, (%rsp) # 64-byte Spill -; AVX512BW-FAST-NEXT: vmovdqa64 (%rsi), %zmm5 -; AVX512BW-FAST-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill -; AVX512BW-FAST-NEXT: vmovdqa64 (%rdx), %zmm2 -; AVX512BW-FAST-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill -; AVX512BW-FAST-NEXT: vmovdqa (%rax), %ymm4 -; AVX512BW-FAST-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX512BW-FAST-NEXT: vmovdqa 32(%rax), %ymm13 +; AVX512BW-FAST-NEXT: vmovdqa64 (%rdi), %zmm7 +; AVX512BW-FAST-NEXT: vmovdqu64 %zmm7, (%rsp) # 64-byte Spill +; AVX512BW-FAST-NEXT: vmovdqa (%rax), %ymm3 +; AVX512BW-FAST-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX512BW-FAST-NEXT: vmovdqa 32(%rax), %ymm11 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} ymm0 = [12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31] -; AVX512BW-FAST-NEXT: vpshufb %ymm0, %ymm4, %ymm1 -; AVX512BW-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [4,5,4,5,5,6,5,6,4,5,4,5,5,6,5,6] -; AVX512BW-FAST-NEXT: # ymm3 = mem[0,1,0,1] -; AVX512BW-FAST-NEXT: vpermw %ymm4, %ymm3, %ymm3 -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm6 -; AVX512BW-FAST-NEXT: vmovdqa (%r9), %ymm15 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm17 = [13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128] -; AVX512BW-FAST-NEXT: vpshufb %ymm17, %ymm15, %ymm7 -; AVX512BW-FAST-NEXT: vmovdqa (%r8), %ymm1 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm20 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128] -; AVX512BW-FAST-NEXT: vpshufb %ymm20, %ymm1, %ymm8 -; AVX512BW-FAST-NEXT: vmovdqa64 %ymm1, %ymm24 -; AVX512BW-FAST-NEXT: vpor %ymm7, %ymm8, %ymm7 -; AVX512BW-FAST-NEXT: vmovdqa64 (%r9), %xmm25 -; AVX512BW-FAST-NEXT: vmovdqa (%r8), %xmm10 -; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm8 = xmm10[8],xmm25[8],xmm10[9],xmm25[9],xmm10[10],xmm25[10],xmm10[11],xmm25[11],xmm10[12],xmm25[12],xmm10[13],xmm25[13],xmm10[14],xmm25[14],xmm10[15],xmm25[15] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm22 +; AVX512BW-FAST-NEXT: vpshufb %ymm0, %ymm3, %ymm1 +; AVX512BW-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [4,5,4,5,5,6,5,6,4,5,4,5,5,6,5,6] +; AVX512BW-FAST-NEXT: # ymm2 = mem[0,1,0,1] +; AVX512BW-FAST-NEXT: vpermw %ymm3, %ymm2, %ymm2 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm4 +; AVX512BW-FAST-NEXT: vmovdqa (%r9), %ymm2 +; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} ymm1 = [13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128] +; AVX512BW-FAST-NEXT: vpshufb %ymm1, %ymm2, %ymm5 +; AVX512BW-FAST-NEXT: vmovdqa %ymm2, %ymm3 +; AVX512BW-FAST-NEXT: vmovdqa (%r8), %ymm2 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm16 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128] +; AVX512BW-FAST-NEXT: vpshufb %ymm16, %ymm2, %ymm6 +; AVX512BW-FAST-NEXT: vmovdqa %ymm2, %ymm12 +; AVX512BW-FAST-NEXT: vpor %ymm5, %ymm6, %ymm5 +; AVX512BW-FAST-NEXT: vmovdqa (%r9), %xmm2 +; AVX512BW-FAST-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX512BW-FAST-NEXT: vmovdqa (%r8), %xmm6 +; AVX512BW-FAST-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1] +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm21 ; AVX512BW-FAST-NEXT: movabsq $2323999253380730912, %r10 # imm = 0x2040810204081020 ; AVX512BW-FAST-NEXT: kmovq %r10, %k1 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm6, %zmm22 {%k1} -; AVX512BW-FAST-NEXT: vmovdqa (%rdx), %ymm1 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm21 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31] -; AVX512BW-FAST-NEXT: vpshufb %ymm21, %ymm1, %ymm6 -; AVX512BW-FAST-NEXT: vmovdqa %ymm1, %ymm7 -; AVX512BW-FAST-NEXT: vmovdqa (%rcx), %ymm1 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm23 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128] -; AVX512BW-FAST-NEXT: vpshufb %ymm23, %ymm1, %ymm11 -; AVX512BW-FAST-NEXT: vmovdqa %ymm1, %ymm8 -; AVX512BW-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX512BW-FAST-NEXT: vpor %ymm6, %ymm11, %ymm6 -; AVX512BW-FAST-NEXT: vmovdqa (%rdx), %xmm14 -; AVX512BW-FAST-NEXT: vmovdqa64 (%rcx), %xmm16 -; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm11 = xmm16[8],xmm14[8],xmm16[9],xmm14[9],xmm16[10],xmm14[10],xmm16[11],xmm14[11],xmm16[12],xmm14[12],xmm16[13],xmm14[13],xmm16[14],xmm14[14],xmm16[15],xmm14[15] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm11 = ymm11[0,1,0,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm6, %zmm11, %zmm26 -; AVX512BW-FAST-NEXT: vmovdqa (%rdi), %ymm11 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm28 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128] -; AVX512BW-FAST-NEXT: vpshufb %ymm28, %ymm11, %ymm6 -; AVX512BW-FAST-NEXT: vmovdqa (%rsi), %ymm12 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm29 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18] -; AVX512BW-FAST-NEXT: vpshufb %ymm29, %ymm12, %ymm18 -; AVX512BW-FAST-NEXT: vporq %ymm6, %ymm18, %ymm6 -; AVX512BW-FAST-NEXT: vmovdqa64 (%rdi), %xmm18 -; AVX512BW-FAST-NEXT: vmovdqa64 (%rsi), %xmm19 -; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm27 = xmm19[8],xmm18[8],xmm19[9],xmm18[9],xmm19[10],xmm18[10],xmm19[11],xmm18[11],xmm19[12],xmm18[12],xmm19[13],xmm18[13],xmm19[14],xmm18[14],xmm19[15],xmm18[15] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm27 = xmm27[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm27 = ymm27[0,1,0,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm6, %zmm27, %zmm6 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm4, %zmm21 {%k1} +; AVX512BW-FAST-NEXT: vmovdqa (%rdx), %ymm2 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm19 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31] +; AVX512BW-FAST-NEXT: vpshufb %ymm19, %ymm2, %ymm4 +; AVX512BW-FAST-NEXT: vmovdqa %ymm2, %ymm5 +; AVX512BW-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX512BW-FAST-NEXT: vmovdqa (%rcx), %ymm2 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm20 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128] +; AVX512BW-FAST-NEXT: vpshufb %ymm20, %ymm2, %ymm9 +; AVX512BW-FAST-NEXT: vmovdqa64 %ymm2, %ymm25 +; AVX512BW-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX512BW-FAST-NEXT: vpor %ymm4, %ymm9, %ymm4 +; AVX512BW-FAST-NEXT: vmovdqa (%rdx), %xmm13 +; AVX512BW-FAST-NEXT: vmovdqa (%rcx), %xmm15 +; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm9 = xmm15[8],xmm13[8],xmm15[9],xmm13[9],xmm15[10],xmm13[10],xmm15[11],xmm13[11],xmm15[12],xmm13[12],xmm15[13],xmm13[13],xmm15[14],xmm13[14],xmm15[15],xmm13[15] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1] +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm4, %zmm9, %zmm22 +; AVX512BW-FAST-NEXT: vmovdqa (%rdi), %ymm14 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm23 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128] +; AVX512BW-FAST-NEXT: vpshufb %ymm23, %ymm14, %ymm4 +; AVX512BW-FAST-NEXT: vmovdqa (%rsi), %ymm10 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm24 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18] +; AVX512BW-FAST-NEXT: vpshufb %ymm24, %ymm10, %ymm17 +; AVX512BW-FAST-NEXT: vporq %ymm4, %ymm17, %ymm4 +; AVX512BW-FAST-NEXT: vmovdqa64 (%rdi), %xmm17 +; AVX512BW-FAST-NEXT: vmovdqa64 (%rsi), %xmm18 +; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm26 = xmm18[8],xmm17[8],xmm18[9],xmm17[9],xmm18[10],xmm17[10],xmm18[11],xmm17[11],xmm18[12],xmm17[12],xmm18[13],xmm17[13],xmm18[14],xmm17[14],xmm18[15],xmm17[15] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm26 = xmm26[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm26 = ymm26[0,1,0,1] +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm4, %zmm26, %zmm8 ; AVX512BW-FAST-NEXT: movabsq $435749860008887046, %r10 # imm = 0x60C183060C18306 ; AVX512BW-FAST-NEXT: kmovq %r10, %k1 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm26, %zmm6 {%k1} +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm22, %zmm8 {%k1} ; AVX512BW-FAST-NEXT: movabsq $4066998693416279096, %r10 # imm = 0x3870E1C3870E1C38 ; AVX512BW-FAST-NEXT: kmovq %r10, %k1 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm22, %zmm6 {%k1} -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm22 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10] -; AVX512BW-FAST-NEXT: vpermw %ymm13, %ymm22, %ymm22 -; AVX512BW-FAST-NEXT: vpshufb %ymm0, %ymm13, %ymm0 -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm22, %zmm0, %zmm22 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%r9), %ymm27 -; AVX512BW-FAST-NEXT: vpshufb %ymm17, %ymm27, %ymm17 -; AVX512BW-FAST-NEXT: vmovdqa 32(%r8), %ymm1 -; AVX512BW-FAST-NEXT: vpshufb %ymm20, %ymm1, %ymm20 -; AVX512BW-FAST-NEXT: vporq %ymm17, %ymm20, %ymm17 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm20 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[20],zero,ymm1[18],zero,ymm1[20,21,20,21],zero,ymm1[19],zero,ymm1[19,20,21,22],zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm20 = ymm20[2,3,2,3] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm26 = ymm27[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm27[20],zero,ymm27[18],zero,zero,zero,zero,ymm27[21],zero,ymm27[19],zero,zero,zero,zero,ymm27[22] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm26 = ymm26[2,3,2,3] -; AVX512BW-FAST-NEXT: vporq %ymm20, %ymm26, %ymm20 -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm20, %zmm17, %zmm26 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm21, %zmm8 {%k1} +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm21 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10] +; AVX512BW-FAST-NEXT: vpermw %ymm11, %ymm21, %ymm21 +; AVX512BW-FAST-NEXT: vpshufb %ymm0, %ymm11, %ymm0 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm21, %zmm0, %zmm21 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%r9), %ymm28 +; AVX512BW-FAST-NEXT: vpshufb %ymm1, %ymm28, %ymm1 +; AVX512BW-FAST-NEXT: vmovdqa 32(%r8), %ymm0 +; AVX512BW-FAST-NEXT: vpshufb %ymm16, %ymm0, %ymm16 +; AVX512BW-FAST-NEXT: vporq %ymm1, %ymm16, %ymm1 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm2 = +; AVX512BW-FAST-NEXT: vpshufb %ymm2, %ymm28, %ymm16 +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm16 = ymm16[2,3,2,3] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm22 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[20],zero,ymm0[18],zero,ymm0[20,21,20,21],zero,ymm0[19],zero,ymm0[19,20,21,22],zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm22 = ymm22[2,3,2,3] +; AVX512BW-FAST-NEXT: vporq %ymm22, %ymm16, %ymm16 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm16, %zmm1, %zmm1 ; AVX512BW-FAST-NEXT: movabsq $145249953336295682, %r10 # imm = 0x204081020408102 ; AVX512BW-FAST-NEXT: kmovq %r10, %k1 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm22, %zmm26 {%k1} -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdx), %ymm22 -; AVX512BW-FAST-NEXT: vpshufb %ymm21, %ymm22, %ymm17 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rcx), %ymm30 -; AVX512BW-FAST-NEXT: vpshufb %ymm23, %ymm30, %ymm20 -; AVX512BW-FAST-NEXT: vporq %ymm17, %ymm20, %ymm17 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm20 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm22[18],zero,ymm22[18,19,20,21],zero,ymm22[19],zero,ymm22[25,26,27,22],zero,ymm22[20],zero +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm21, %zmm1 {%k1} +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdx), %ymm26 +; AVX512BW-FAST-NEXT: vpshufb %ymm19, %ymm26, %ymm16 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rcx), %ymm29 +; AVX512BW-FAST-NEXT: vpshufb %ymm20, %ymm29, %ymm19 +; AVX512BW-FAST-NEXT: vporq %ymm16, %ymm19, %ymm16 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm2 = +; AVX512BW-FAST-NEXT: vpshufb %ymm2, %ymm29, %ymm20 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm20 = ymm20[2,3,2,3] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm21 = ymm30[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm30[18],zero,zero,zero,zero,ymm30[21],zero,ymm30[19],zero,zero,zero,zero,ymm30[22],zero,ymm30[20] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm21 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm26[18],zero,ymm26[18,19,20,21],zero,ymm26[19],zero,ymm26[25,26,27,22],zero,ymm26[20],zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm21 = ymm21[2,3,2,3] -; AVX512BW-FAST-NEXT: vporq %ymm20, %ymm21, %ymm20 -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm20, %zmm17, %zmm21 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %ymm31 -; AVX512BW-FAST-NEXT: vpshufb %ymm28, %ymm31, %ymm17 -; AVX512BW-FAST-NEXT: vmovdqa 32(%rsi), %ymm0 -; AVX512BW-FAST-NEXT: vpshufb %ymm29, %ymm0, %ymm20 -; AVX512BW-FAST-NEXT: vporq %ymm17, %ymm20, %ymm17 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm20 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm31[18,19,20,21],zero,ymm31[19],zero,ymm31[21,20,21,22],zero,ymm31[20],zero,ymm31[22,23] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm20 = ymm20[2,3,2,3] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm23 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm0[21],zero,ymm0[19],zero,zero,zero,zero,ymm0[22],zero,ymm0[20],zero,zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm23 = ymm23[2,3,2,3] -; AVX512BW-FAST-NEXT: vporq %ymm20, %ymm23, %ymm20 -; AVX512BW-FAST-NEXT: vmovdqa64 (%rcx), %zmm23 -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm20, %zmm17, %zmm17 -; AVX512BW-FAST-NEXT: vmovdqa64 (%r8), %zmm3 -; AVX512BW-FAST-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill +; AVX512BW-FAST-NEXT: vporq %ymm21, %ymm20, %ymm20 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm20, %zmm16, %zmm20 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %ymm30 +; AVX512BW-FAST-NEXT: vpshufb %ymm23, %ymm30, %ymm16 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rsi), %ymm31 +; AVX512BW-FAST-NEXT: vpshufb %ymm24, %ymm31, %ymm21 +; AVX512BW-FAST-NEXT: vporq %ymm16, %ymm21, %ymm16 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm21 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm30[18,19,20,21],zero,ymm30[19],zero,ymm30[21,20,21,22],zero,ymm30[20],zero,ymm30[22,23] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm21 = ymm21[2,3,2,3] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm22 = ymm31[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm31[21],zero,ymm31[19],zero,zero,zero,zero,ymm31[22],zero,ymm31[20],zero,zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm22 = ymm22[2,3,2,3] +; AVX512BW-FAST-NEXT: vporq %ymm21, %ymm22, %ymm21 +; AVX512BW-FAST-NEXT: vmovdqa64 (%rsi), %zmm24 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm21, %zmm16, %zmm16 +; AVX512BW-FAST-NEXT: vmovdqa64 (%rdx), %zmm22 ; AVX512BW-FAST-NEXT: movabsq $3485998880071096368, %r10 # imm = 0x3060C183060C1830 ; AVX512BW-FAST-NEXT: kmovq %r10, %k2 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm21, %zmm17 {%k2} -; AVX512BW-FAST-NEXT: vmovdqa64 (%r9), %zmm21 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm20, %zmm16 {%k2} +; AVX512BW-FAST-NEXT: vmovdqa64 (%rcx), %zmm23 ; AVX512BW-FAST-NEXT: movabsq $-4357498600088870461, %r10 # imm = 0xC3870E1C3870E1C3 ; AVX512BW-FAST-NEXT: kmovq %r10, %k1 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm26, %zmm17 {%k1} -; AVX512BW-FAST-NEXT: vmovdqa64 (%rax), %zmm26 -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm22 = zmm22[0,1,2,3],zmm23[4,5,6,7] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm22 = zmm22[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,zmm22[23],zero,zmm22[21,22,23,26],zero,zmm22[24],zero,zmm22[28,29,26,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,59],zero,zero,zero,zero,zmm22[62],zero,zmm22[60],zero,zero,zero,zero,zmm22[63],zero,zmm22[61],zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm22 = zmm22[2,3,2,3,6,7,6,7] -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm28 = zmm30[0,1,2,3],zmm2[4,5,6,7] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm28 = zmm28[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm28[25],zero,zmm28[23],zero,zero,zero,zero,zmm28[26],zero,zmm28[24],zero,zero,zero,zero,zmm28[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm28[62],zero,zmm28[60],zero,zero,zero,zero,zmm28[63],zero,zmm28[61],zero,zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm28 = zmm28[2,3,2,3,6,7,6,7] -; AVX512BW-FAST-NEXT: vporq %zmm22, %zmm28, %zmm29 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %xmm28 -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm9[4,5,6,7] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,zmm0[23],zero,zero,zero,zero,zmm0[26],zero,zmm0[24],zero,zero,zero,zero,zmm0[27],zero,zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,60,61,62],zero,zmm0[60],zero,zmm0[62,63,62,63],zero,zmm0[61],zero,zmm0[63,60,61] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7] -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm22 = zmm31[0,1,2,3],zmm5[4,5,6,7] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm22 = zmm22[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm22[23],zero,zero,zero,zero,zmm22[26],zero,zmm22[24],zero,zero,zero,zero,zmm22[27],zero,zmm22[25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zmm22[62],zero,zmm22[60],zero,zero,zero,zero,zmm22[63],zero,zmm22[61],zero,zero,zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm22 = zmm22[2,3,2,3,6,7,6,7] -; AVX512BW-FAST-NEXT: vporq %zmm0, %zmm22, %zmm22 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rsi), %xmm30 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm16 {%k1} +; AVX512BW-FAST-NEXT: vmovdqa64 (%r8), %zmm27 +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm26[0,1,2,3],zmm23[4,5,6,7] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,zmm1[23],zero,zmm1[21,22,23,26],zero,zmm1[24],zero,zmm1[28,29,26,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,59],zero,zero,zero,zero,zmm1[62],zero,zmm1[60],zero,zero,zero,zero,zmm1[63],zero,zmm1[61],zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7] +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm29[0,1,2,3],zmm22[4,5,6,7] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm20 = zmm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm20[25],zero,zmm20[23],zero,zero,zero,zero,zmm20[26],zero,zmm20[24],zero,zero,zero,zero,zmm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm20[62],zero,zmm20[60],zero,zero,zero,zero,zmm20[63],zero,zmm20[61],zero,zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm20 = zmm20[2,3,2,3,6,7,6,7] +; AVX512BW-FAST-NEXT: vporq %zmm1, %zmm20, %zmm1 +; AVX512BW-FAST-NEXT: vmovdqa64 (%r9), %zmm26 +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm31[0,1,2,3],zmm7[4,5,6,7] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm20 = zmm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,zmm20[23],zero,zero,zero,zero,zmm20[26],zero,zmm20[24],zero,zero,zero,zero,zmm20[27],zero,zmm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,60,61,62],zero,zmm20[60],zero,zmm20[62,63,62,63],zero,zmm20[61],zero,zmm20[63,60,61] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm20 = zmm20[2,3,2,3,6,7,6,7] +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm29 = zmm30[0,1,2,3],zmm24[4,5,6,7] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm29 = zmm29[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm29[23],zero,zero,zero,zero,zmm29[26],zero,zmm29[24],zero,zero,zero,zero,zmm29[27],zero,zmm29[25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zmm29[62],zero,zmm29[60],zero,zero,zero,zero,zmm29[63],zero,zmm29[61],zero,zero,zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm29 = zmm29[2,3,2,3,6,7,6,7] +; AVX512BW-FAST-NEXT: vporq %zmm20, %zmm29, %zmm20 +; AVX512BW-FAST-NEXT: vmovdqa64 (%rax), %zmm29 ; AVX512BW-FAST-NEXT: movabsq $1742999440035548184, %rax # imm = 0x183060C183060C18 ; AVX512BW-FAST-NEXT: kmovq %rax, %k1 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm29, %zmm22 {%k1} -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm21[4,5,6,7] +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm20 {%k1} +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %xmm31 +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm26[4,5,6,7] ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,zmm0[23],zero,zmm0[23,24,25,26],zero,zmm0[24],zero,zmm0[30,31,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,61],zero,zmm0[59],zero,zero,zero,zero,zmm0[62],zero,zmm0[60],zero,zero,zero,zero,zmm0[63],zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7] -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm27[0,1,2,3],zmm3[4,5,6,7] +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm28[0,1,2,3],zmm27[4,5,6,7] ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm1[25],zero,zmm1[23],zero,zero,zero,zero,zmm1[26],zero,zmm1[24],zero,zero,zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm1[59],zero,zero,zero,zero,zmm1[62],zero,zmm1[60],zero,zero,zero,zero,zmm1[63],zero,zmm1[61] ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7] ; AVX512BW-FAST-NEXT: vporq %zmm0, %zmm1, %zmm0 ; AVX512BW-FAST-NEXT: movabsq $6971997760142192736, %rax # imm = 0x60C183060C183060 ; AVX512BW-FAST-NEXT: kmovq %rax, %k1 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm0, %zmm22 {%k1} +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm0, %zmm20 {%k1} ; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm0 = [11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12,62,61,62,63,63,62,62,63,62,61,62,63,63,62,62,63] -; AVX512BW-FAST-NEXT: vpermi2w %zmm26, %zmm13, %zmm0 +; AVX512BW-FAST-NEXT: vpermi2w %zmm29, %zmm11, %zmm0 ; AVX512BW-FAST-NEXT: movabsq $-9150747060186627967, %rax # imm = 0x8102040810204081 ; AVX512BW-FAST-NEXT: kmovq %rax, %k3 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm0, %zmm22 {%k3} -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,30],zero,ymm11[28],zero,ymm11[30,31,30,31],zero,ymm11[29],zero,ymm11[31,28,29] +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm0, %zmm20 {%k3} +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,30],zero,ymm14[28],zero,ymm14[30,31,30,31],zero,ymm14[29],zero,ymm14[31,28,29] ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm12[30],zero,ymm12[28],zero,zero,zero,zero,ymm12[31],zero,ymm12[29],zero,zero,zero +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm10[30],zero,ymm10[28],zero,zero,zero,zero,ymm10[31],zero,ymm10[29],zero,zero,zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3] ; AVX512BW-FAST-NEXT: vpor %ymm0, %ymm1, %ymm0 -; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm28[0],xmm30[0],xmm28[1],xmm30[1],xmm28[2],xmm30[2],xmm28[3],xmm30[3],xmm28[4],xmm30[4],xmm28[5],xmm30[5],xmm28[6],xmm30[6],xmm28[7],xmm30[7] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm3 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,zero,zero,ymm8[30],zero,ymm8[28],zero,zero,zero,zero,ymm8[31],zero,ymm8[29],zero +; AVX512BW-FAST-NEXT: vmovdqa 32(%rsi), %xmm7 +; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm31[0],xmm7[0],xmm31[1],xmm7[1],xmm31[2],xmm7[2],xmm31[3],xmm7[3],xmm31[4],xmm7[4],xmm31[5],xmm7[5],xmm31[6],xmm7[6],xmm31[7],xmm7[7] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1] +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm6 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm25[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,zero,zero,ymm25[30],zero,ymm25[28],zero,zero,zero,zero,ymm25[31],zero,ymm25[29],zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm27 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm7[30],zero,ymm7[28],zero,zero,zero,zero,ymm7[31],zero,ymm7[29],zero,zero -; AVX512BW-FAST-NEXT: vmovdqa64 %ymm7, %ymm20 -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm27 = ymm27[2,3,2,3] -; AVX512BW-FAST-NEXT: vporq %ymm0, %ymm27, %ymm27 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdx), %xmm31 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm28 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm5[30],zero,ymm5[28],zero,zero,zero,zero,ymm5[31],zero,ymm5[29],zero,zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm28 = ymm28[2,3,2,3] +; AVX512BW-FAST-NEXT: vporq %ymm0, %ymm28, %ymm28 +; AVX512BW-FAST-NEXT: vmovdqa 32(%rdx), %xmm2 ; AVX512BW-FAST-NEXT: vmovdqa 32(%rcx), %xmm1 -; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm31[0],xmm1[0],xmm31[1],xmm1[1],xmm31[2],xmm1[2],xmm31[3],xmm1[3],xmm31[4],xmm1[4],xmm31[5],xmm1[5],xmm31[6],xmm1[6],xmm31[7],xmm1[7] -; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm7 = <4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9> -; AVX512BW-FAST-NEXT: vpshufb %xmm7, %xmm0, %xmm0 +; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm9 = <4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9> +; AVX512BW-FAST-NEXT: vpshufb %xmm9, %xmm0, %xmm0 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm27, %zmm27 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm3, %zmm27 {%k2} -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm15[27],zero,zero,zero,zero,ymm15[30],zero,ymm15[28],zero,zero,zero,zero,ymm15[31],zero +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm28, %zmm28 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm6, %zmm28 {%k2} +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm3[27],zero,zero,zero,zero,ymm3[30],zero,ymm3[28],zero,zero,zero,zero,ymm3[31],zero +; AVX512BW-FAST-NEXT: vmovdqa64 %ymm3, %ymm30 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm3 = ymm24[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm24[27],zero,zero,zero,zero,ymm24[30],zero,ymm24[28],zero,zero,zero,zero,ymm24[31],zero,ymm24[29] -; AVX512BW-FAST-NEXT: vmovdqa64 %ymm24, %ymm9 -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,2,3] -; AVX512BW-FAST-NEXT: vpor %ymm0, %ymm3, %ymm2 -; AVX512BW-FAST-NEXT: vmovdqa 32(%r9), %xmm4 -; AVX512BW-FAST-NEXT: vmovdqa 32(%r8), %xmm3 -; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] -; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm8 = -; AVX512BW-FAST-NEXT: vpshufb %xmm8, %xmm0, %xmm0 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm6 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm12[27],zero,zero,zero,zero,ymm12[30],zero,ymm12[28],zero,zero,zero,zero,ymm12[31],zero,ymm12[29] +; AVX512BW-FAST-NEXT: vmovdqa64 %ymm12, %ymm21 +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,3,2,3] +; AVX512BW-FAST-NEXT: vpor %ymm0, %ymm6, %ymm3 +; AVX512BW-FAST-NEXT: vmovdqa 32(%r9), %xmm5 +; AVX512BW-FAST-NEXT: vmovdqa 32(%r8), %xmm4 +; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7] +; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm12 = +; AVX512BW-FAST-NEXT: vpshufb %xmm12, %xmm0, %xmm0 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm2 = [14,13,14,15,15,14,14,15,14,13,14,15,15,14,14,15,17,17,16,16,17,17,16,16,20,21,17,17,17,17,16,16] -; AVX512BW-FAST-NEXT: vpermw %zmm26, %zmm2, %zmm2 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm3 = [14,13,14,15,15,14,14,15,14,13,14,15,15,14,14,15,17,17,16,16,17,17,16,16,20,21,17,17,17,17,16,16] +; AVX512BW-FAST-NEXT: vpermw %zmm29, %zmm3, %zmm3 ; AVX512BW-FAST-NEXT: movabsq $580999813345182728, %rax # imm = 0x810204081020408 ; AVX512BW-FAST-NEXT: kmovq %rax, %k2 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm2, %zmm0 {%k2} +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm3, %zmm0 {%k2} ; AVX512BW-FAST-NEXT: movabsq $1016749673354069774, %rax # imm = 0xE1C3870E1C3870E ; AVX512BW-FAST-NEXT: kmovq %rax, %k2 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm0, %zmm27 {%k2} -; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm5 = -; AVX512BW-FAST-NEXT: vpshufb %xmm5, %xmm1, %xmm2 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm0, %zmm28 {%k2} +; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm6 = +; AVX512BW-FAST-NEXT: vpshufb %xmm6, %xmm1, %xmm3 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm0 = -; AVX512BW-FAST-NEXT: vpshufb %xmm0, %xmm31, %xmm24 -; AVX512BW-FAST-NEXT: vporq %xmm2, %xmm24, %xmm2 -; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm31[8],xmm1[9],xmm31[9],xmm1[10],xmm31[10],xmm1[11],xmm31[11],xmm1[12],xmm31[12],xmm1[13],xmm31[13],xmm1[14],xmm31[14],xmm1[15],xmm31[15] +; AVX512BW-FAST-NEXT: vpshufb %xmm0, %xmm2, %xmm19 +; AVX512BW-FAST-NEXT: vporq %xmm3, %xmm19, %xmm3 +; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7] -; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm2, %zmm1 +; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm3, %zmm1 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm2 = -; AVX512BW-FAST-NEXT: vpshufb %xmm2, %xmm30, %xmm24 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} xmm31 = -; AVX512BW-FAST-NEXT: vpshufb %xmm31, %xmm28, %xmm29 -; AVX512BW-FAST-NEXT: vporq %xmm24, %xmm29, %xmm24 -; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm28 = xmm30[8],xmm28[8],xmm30[9],xmm28[9],xmm30[10],xmm28[10],xmm30[11],xmm28[11],xmm30[12],xmm28[12],xmm30[13],xmm28[13],xmm30[14],xmm28[14],xmm30[15],xmm28[15] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm28 = xmm28[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u] -; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm28, %zmm24, %zmm24 +; AVX512BW-FAST-NEXT: vpshufb %xmm2, %xmm7, %xmm3 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} xmm19 = +; AVX512BW-FAST-NEXT: vpshufb %xmm19, %xmm31, %xmm25 +; AVX512BW-FAST-NEXT: vporq %xmm3, %xmm25, %xmm3 +; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm7[8],xmm31[8],xmm7[9],xmm31[9],xmm7[10],xmm31[10],xmm7[11],xmm31[11],xmm7[12],xmm31[12],xmm7[13],xmm31[13],xmm7[14],xmm31[14],xmm7[15],xmm31[15] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u] +; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm7, %zmm3, %zmm3 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm1[0,1,0,1,4,5,4,5] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm24 = zmm24[0,1,0,1,4,5,4,5] -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm24 {%k1} +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm25 = zmm3[0,1,0,1,4,5,4,5] +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm25 {%k1} ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = <128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6> -; AVX512BW-FAST-NEXT: vpshufb %xmm1, %xmm4, %xmm28 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} xmm29 = <4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128> -; AVX512BW-FAST-NEXT: vpshufb %xmm29, %xmm3, %xmm30 -; AVX512BW-FAST-NEXT: vporq %xmm28, %xmm30, %xmm28 -; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10] -; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm3, %zmm28, %zmm3 +; AVX512BW-FAST-NEXT: vpshufb %xmm1, %xmm5, %xmm3 +; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm7 = <4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128> +; AVX512BW-FAST-NEXT: vpshufb %xmm7, %xmm4, %xmm31 +; AVX512BW-FAST-NEXT: vporq %xmm3, %xmm31, %xmm3 +; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10] +; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm4, %zmm3, %zmm3 ; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm4 = [2,2,2,4,2,2,2,4,3,3,3,3,2,2,2,4,52,53,52,53,53,54,53,54,52,53,52,53,53,54,53,54] -; AVX512BW-FAST-NEXT: vpermi2w %zmm26, %zmm13, %zmm4 +; AVX512BW-FAST-NEXT: vpermi2w %zmm29, %zmm11, %zmm4 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm3 = zmm3[0,1,0,1,4,5,4,5] ; AVX512BW-FAST-NEXT: movabsq $290499906672591364, %rax # imm = 0x408102040810204 ; AVX512BW-FAST-NEXT: kmovq %rax, %k2 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm4, %zmm3 {%k2} ; AVX512BW-FAST-NEXT: movabsq $-8714997200177740921, %rax # imm = 0x870E1C3870E1C387 ; AVX512BW-FAST-NEXT: kmovq %rax, %k2 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm3, %zmm24 {%k2} -; AVX512BW-FAST-NEXT: vpshufb %xmm5, %xmm16, %xmm3 -; AVX512BW-FAST-NEXT: vpshufb %xmm0, %xmm14, %xmm0 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm3, %zmm25 {%k2} +; AVX512BW-FAST-NEXT: vpshufb %xmm6, %xmm15, %xmm3 +; AVX512BW-FAST-NEXT: vpshufb %xmm0, %xmm13, %xmm0 ; AVX512BW-FAST-NEXT: vpor %xmm3, %xmm0, %xmm0 -; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm14[0],xmm16[0],xmm14[1],xmm16[1],xmm14[2],xmm16[2],xmm14[3],xmm16[3],xmm14[4],xmm16[4],xmm14[5],xmm16[5],xmm14[6],xmm16[6],xmm14[7],xmm16[7] -; AVX512BW-FAST-NEXT: vpshufb %xmm7, %xmm3, %xmm3 +; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3],xmm13[4],xmm15[4],xmm13[5],xmm15[5],xmm13[6],xmm15[6],xmm13[7],xmm15[7] +; AVX512BW-FAST-NEXT: vpshufb %xmm9, %xmm3, %xmm3 ; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm3, %zmm0 -; AVX512BW-FAST-NEXT: vpshufb %xmm2, %xmm19, %xmm2 -; AVX512BW-FAST-NEXT: vpshufb %xmm31, %xmm18, %xmm3 +; AVX512BW-FAST-NEXT: vpshufb %xmm2, %xmm18, %xmm2 +; AVX512BW-FAST-NEXT: vpshufb %xmm19, %xmm17, %xmm3 ; AVX512BW-FAST-NEXT: vpor %xmm2, %xmm3, %xmm2 -; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm18[0],xmm19[0],xmm18[1],xmm19[1],xmm18[2],xmm19[2],xmm18[3],xmm19[3],xmm18[4],xmm19[4],xmm18[5],xmm19[5],xmm18[6],xmm19[6],xmm18[7],xmm19[7] +; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3],xmm17[4],xmm18[4],xmm17[5],xmm18[5],xmm17[6],xmm18[6],xmm17[7],xmm18[7] ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5] ; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm2, %zmm3, %zmm2 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm3 = zmm0[0,1,0,1,4,5,4,5] @@ -8666,14 +8667,16 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FAST-NEXT: movabsq $871499720017774092, %rax # imm = 0xC183060C183060C ; AVX512BW-FAST-NEXT: kmovq %rax, %k2 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm3, %zmm0 {%k2} -; AVX512BW-FAST-NEXT: vpshufb %xmm1, %xmm25, %xmm1 -; AVX512BW-FAST-NEXT: vpshufb %xmm29, %xmm10, %xmm2 +; AVX512BW-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; AVX512BW-FAST-NEXT: vpshufb %xmm1, %xmm3, %xmm1 +; AVX512BW-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload +; AVX512BW-FAST-NEXT: vpshufb %xmm7, %xmm4, %xmm2 ; AVX512BW-FAST-NEXT: vpor %xmm1, %xmm2, %xmm1 -; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm10[0],xmm25[0],xmm10[1],xmm25[1],xmm10[2],xmm25[2],xmm10[3],xmm25[3],xmm10[4],xmm25[4],xmm10[5],xmm25[5],xmm10[6],xmm25[6],xmm10[7],xmm25[7] -; AVX512BW-FAST-NEXT: vpshufb %xmm8, %xmm2, %xmm2 +; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7] +; AVX512BW-FAST-NEXT: vpshufb %xmm12, %xmm2, %xmm2 ; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm2, %zmm1 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm1[0,1,0,1,4,5,4,5] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm2 # 32-byte Folded Reload +; AVX512BW-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm2 # 32-byte Folded Reload ; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm3 = [1,1,0,0,1,1,0,0,4,5,1,1,1,1,0,0,18,18,18,20,18,18,18,20,19,19,19,19,18,18,18,20] ; AVX512BW-FAST-NEXT: vpermw %zmm2, %zmm3, %zmm3 ; AVX512BW-FAST-NEXT: movabsq $4647998506761461824, %rax # imm = 0x4081020408102040 @@ -8683,29 +8686,26 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FAST-NEXT: kmovq %rax, %k2 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm0 {%k2} ; AVX512BW-FAST-NEXT: vmovdqu64 (%rsp), %zmm1 # 64-byte Reload -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm12, %zmm1, %zmm1 -; AVX512BW-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm11, %zmm3, %zmm3 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm10, %zmm1, %zmm1 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm14, %zmm24, %zmm3 ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21],zero,zmm1[19],zero,zmm1[21,20,21,22],zero,zmm1[20],zero,zmm1[22,23,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,57],zero,zmm1[55],zero,zero,zero,zero,zmm1[58],zero,zmm1[56],zero,zero,zero,zero,zmm1[59],zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7] ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm3 = zmm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm3[21],zero,zmm3[19],zero,zero,zero,zero,zmm3[22],zero,zmm3[20],zero,zero,zmm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm3[55],zero,zero,zero,zero,zmm3[58],zero,zmm3[56],zero,zero,zero,zero,zmm3[59],zero,zmm3[57] ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm3 = zmm3[2,3,2,3,6,7,6,7] ; AVX512BW-FAST-NEXT: vporq %zmm1, %zmm3, %zmm1 -; AVX512BW-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm20, %zmm3, %zmm3 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm22, %zmm3 # 32-byte Folded Reload ; AVX512BW-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm23, %zmm4 # 32-byte Folded Reload +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm4 = zmm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm4[18],zero,zero,zero,zero,zmm4[21],zero,zmm4[19],zero,zero,zero,zero,zmm4[22],zero,zmm4[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm4[57],zero,zmm4[55],zero,zero,zero,zero,zmm4[58],zero,zmm4[56],zero,zero,zero,zero ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm3 = zmm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,zmm3[18,19,20,21],zero,zmm3[19],zero,zmm3[25,26,27,22],zero,zmm3[20],zero,zmm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,56,57],zero,zmm3[55],zero,zmm3[53,54,55,58],zero,zmm3[56],zero,zmm3[60,61,58,59] ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm3 = zmm3[2,3,2,3,6,7,6,7] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm4 = zmm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm4[18],zero,zero,zero,zero,zmm4[21],zero,zmm4[19],zero,zero,zero,zero,zmm4[22],zero,zmm4[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm4[57],zero,zmm4[55],zero,zero,zero,zero,zmm4[58],zero,zmm4[56],zero,zero,zero,zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm4 = zmm4[2,3,2,3,6,7,6,7] ; AVX512BW-FAST-NEXT: vporq %zmm3, %zmm4, %zmm3 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm3 {%k1} -; AVX512BW-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm9, %zmm1, %zmm1 -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm15, %zmm21, %zmm4 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm21, %zmm27, %zmm1 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm30, %zmm26, %zmm4 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm4 = zmm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm4[20],zero,zmm4[18],zero,zero,zero,zero,zmm4[21],zero,zmm4[19],zero,zero,zero,zero,zmm4[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm4[57],zero,zmm4[55],zero,zero,zero,zero,zmm4[58],zero,zmm4[56],zero,zero ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,zmm1[18],zero,zmm1[20,21,20,21],zero,zmm1[19],zero,zmm1[19,20,21,22],zero,zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,56,57,56,57],zero,zmm1[55],zero,zmm1[55,56,57,58],zero,zmm1[56],zero,zmm1[62,63] ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm4 = zmm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm4[20],zero,zmm4[18],zero,zero,zero,zero,zmm4[21],zero,zmm4[19],zero,zero,zero,zero,zmm4[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm4[57],zero,zmm4[55],zero,zero,zero,zero,zmm4[58],zero,zmm4[56],zero,zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm4 = zmm4[2,3,2,3,6,7,6,7] ; AVX512BW-FAST-NEXT: vporq %zmm1, %zmm4, %zmm1 ; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm4 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,27,29,28,27,28,29,29,28,27,29,28,27,28,29,29,28] @@ -8718,13 +8718,13 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm3 {%k1} ; AVX512BW-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512BW-FAST-NEXT: vmovdqa64 %zmm3, 128(%rax) -; AVX512BW-FAST-NEXT: vmovdqa64 %zmm17, 320(%rax) +; AVX512BW-FAST-NEXT: vmovdqa64 %zmm16, 320(%rax) ; AVX512BW-FAST-NEXT: vmovdqa64 %zmm0, (%rax) -; AVX512BW-FAST-NEXT: vmovdqa64 %zmm24, 256(%rax) -; AVX512BW-FAST-NEXT: vmovdqa64 %zmm27, 192(%rax) -; AVX512BW-FAST-NEXT: vmovdqa64 %zmm6, 64(%rax) -; AVX512BW-FAST-NEXT: vmovdqa64 %zmm22, 384(%rax) -; AVX512BW-FAST-NEXT: addq $200, %rsp +; AVX512BW-FAST-NEXT: vmovdqa64 %zmm25, 256(%rax) +; AVX512BW-FAST-NEXT: vmovdqa64 %zmm28, 192(%rax) +; AVX512BW-FAST-NEXT: vmovdqa64 %zmm8, 64(%rax) +; AVX512BW-FAST-NEXT: vmovdqa64 %zmm20, 384(%rax) +; AVX512BW-FAST-NEXT: addq $72, %rsp ; AVX512BW-FAST-NEXT: vzeroupper ; AVX512BW-FAST-NEXT: retq %in.vec0 = load <64 x i8>, ptr %in.vecptr0, align 64 diff --git a/llvm/test/CodeGen/X86/vector-rotate-128.ll b/llvm/test/CodeGen/X86/vector-rotate-128.ll index 33f7a4e42b7f70cf1ed9623346a503fb99c0d43f..43c9be2dc6f9769279db6b895a268e2adac6015d 100644 --- a/llvm/test/CodeGen/X86/vector-rotate-128.ll +++ b/llvm/test/CodeGen/X86/vector-rotate-128.ll @@ -1573,7 +1573,7 @@ define <16 x i8> @splatconstant_rotate_v16i8(<16 x i8> %a) nounwind { ; AVX512NOVLX: # %bb.0: ; AVX512NOVLX-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512NOVLX-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512NOVLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512NOVLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512NOVLX-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512NOVLX-NEXT: vzeroupper ; AVX512NOVLX-NEXT: retq @@ -1582,7 +1582,7 @@ define <16 x i8> @splatconstant_rotate_v16i8(<16 x i8> %a) nounwind { ; AVX512VLX: # %bb.0: ; AVX512VLX-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VLX-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VLX-NEXT: retq ; ; XOP-LABEL: splatconstant_rotate_v16i8: @@ -1811,7 +1811,7 @@ define <16 x i8> @splatconstant_rotate_mask_v16i8(<16 x i8> %a) nounwind { ; AVX512NOVLX: # %bb.0: ; AVX512NOVLX-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512NOVLX-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512NOVLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512NOVLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512NOVLX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ; AVX512NOVLX-NEXT: vzeroupper ; AVX512NOVLX-NEXT: retq @@ -1820,7 +1820,7 @@ define <16 x i8> @splatconstant_rotate_mask_v16i8(<16 x i8> %a) nounwind { ; AVX512VLX: # %bb.0: ; AVX512VLX-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VLX-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VLX-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0 ; AVX512VLX-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll index 7047d5e3131a76f608f17a63fc16578ca089bfe2..c55335f8495697976bb25b5490f18e0f0d05e368 100644 --- a/llvm/test/CodeGen/X86/vector-rotate-256.ll +++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll @@ -392,12 +392,12 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm3 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm3 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm3 ; AVX512F-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512F-NEXT: vpsllw $2, %ymm0, %ymm2 ; AVX512F-NEXT: vpsrlw $6, %ymm0, %ymm3 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm3 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm3 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm2 @@ -412,12 +412,12 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm3 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3 ; AVX512VL-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsllw $2, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $6, %ymm0, %ymm3 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm3 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $7, %ymm0, %ymm2 @@ -1404,7 +1404,7 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind { ; AVX512NOVLX: # %bb.0: ; AVX512NOVLX-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512NOVLX-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512NOVLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512NOVLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512NOVLX-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512NOVLX-NEXT: retq ; @@ -1412,7 +1412,7 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind { ; AVX512VLX: # %bb.0: ; AVX512VLX-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VLX-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VLX-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_rotate_v32i8: @@ -1667,7 +1667,7 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind { ; AVX512NOVLX: # %bb.0: ; AVX512NOVLX-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512NOVLX-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512NOVLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512NOVLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512NOVLX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 ; AVX512NOVLX-NEXT: retq ; @@ -1675,7 +1675,7 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind { ; AVX512VLX: # %bb.0: ; AVX512VLX-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VLX-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VLX-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0 ; AVX512VLX-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vector-rotate-512.ll b/llvm/test/CodeGen/X86/vector-rotate-512.ll index 6504c3e6353e9e854081fdedb06c4715b1412296..29afbf4c62ef5ad434fb85147b7a5324c00be540 100644 --- a/llvm/test/CodeGen/X86/vector-rotate-512.ll +++ b/llvm/test/CodeGen/X86/vector-rotate-512.ll @@ -135,15 +135,15 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm3 ; AVX512F-NEXT: vpsllw $4, %ymm2, %ymm4 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm5 = [17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520] -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm5, %zmm4 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm5 = [4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160] +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm5, %zmm4 ; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm3 ; AVX512F-NEXT: vpsllw $5, %ymm3, %ymm3 ; AVX512F-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512F-NEXT: vpsrlw $6, %ymm2, %ymm4 ; AVX512F-NEXT: vpsllw $2, %ymm2, %ymm6 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm7 = [18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596] -; AVX512F-NEXT: vpternlogq $226, %zmm4, %zmm7, %zmm6 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm7 = [4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268] +; AVX512F-NEXT: vpternlogd $226, %zmm4, %zmm7, %zmm6 ; AVX512F-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512F-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512F-NEXT: vpsrlw $7, %ymm2, %ymm4 @@ -155,12 +155,12 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { ; AVX512F-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm3 ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm4 -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm5, %zmm4 +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm5, %zmm4 ; AVX512F-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $6, %ymm0, %ymm3 ; AVX512F-NEXT: vpsllw $2, %ymm0, %ymm4 -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm7, %zmm4 +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm7, %zmm4 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm3 @@ -177,15 +177,15 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { ; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm3 ; AVX512VL-NEXT: vpsllw $4, %ymm2, %ymm4 -; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm5 = [17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520] -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm5, %ymm4 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm5 = [4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160] +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm5, %ymm4 ; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm3 ; AVX512VL-NEXT: vpsllw $5, %ymm3, %ymm3 ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsrlw $6, %ymm2, %ymm4 ; AVX512VL-NEXT: vpsllw $2, %ymm2, %ymm6 -; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm7 = [18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596] -; AVX512VL-NEXT: vpternlogq $226, %ymm4, %ymm7, %ymm6 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm7 = [4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268] +; AVX512VL-NEXT: vpternlogd $226, %ymm4, %ymm7, %ymm6 ; AVX512VL-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsrlw $7, %ymm2, %ymm4 @@ -196,12 +196,12 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3 ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm4 -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm5, %ymm4 +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm5, %ymm4 ; AVX512VL-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $6, %ymm0, %ymm3 ; AVX512VL-NEXT: vpsllw $2, %ymm0, %ymm4 -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm7, %ymm4 +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm7, %ymm4 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $7, %ymm0, %ymm3 @@ -754,7 +754,7 @@ define <64 x i8> @splatconstant_rotate_v64i8(<64 x i8> %a) nounwind { ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: splatconstant_rotate_v64i8: @@ -766,35 +766,35 @@ define <64 x i8> @splatconstant_rotate_v64i8(<64 x i8> %a) nounwind { ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_rotate_v64i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: retq ; ; AVX512VLBW-LABEL: splatconstant_rotate_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLBW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_rotate_v64i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: retq ; ; AVX512VLVBMI2-LABEL: splatconstant_rotate_v64i8: ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLVBMI2-NEXT: retq %shl = shl <64 x i8> %a, %lshr = lshr <64 x i8> %a, @@ -902,7 +902,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind { ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0 ; AVX512F-NEXT: retq ; @@ -915,7 +915,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind { ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0 ; AVX512VL-NEXT: retq ; @@ -923,7 +923,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind { ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0 ; AVX512BW-NEXT: retq ; @@ -931,7 +931,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind { ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLBW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0 ; AVX512VLBW-NEXT: retq ; @@ -939,7 +939,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind { ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0 ; AVX512VBMI2-NEXT: retq ; @@ -947,7 +947,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind { ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0 ; AVX512VLVBMI2-NEXT: retq %shl = shl <64 x i8> %a, diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll index 938fba0490b557fc27796940217b1699b4ec7851..e6b5998d965f067e4d5720573c8acfc54042f67d 100644 --- a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll +++ b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll @@ -1160,9 +1160,9 @@ define <16 x i8> @splatvar_modulo_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwi ; ; XOPAVX1-LABEL: splatvar_modulo_shift_v16i8: ; XOPAVX1: # %bb.0: -; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm1 ; XOPAVX1-NEXT: vpshab %xmm1, %xmm0, %xmm0 ; XOPAVX1-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll index 5a70e5d4a2b560d8793cde1b0caa8f54cdb881c5..6d19a81d9fd782bc4b25af38579e4995445c8103 100644 --- a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll +++ b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll @@ -1265,9 +1265,9 @@ define <32 x i8> @splatvar_modulo_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwi ; ; XOPAVX1-LABEL: splatvar_modulo_shift_v32i8: ; XOPAVX1: # %bb.0: -; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm1 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 ; XOPAVX1-NEXT: vpshab %xmm1, %xmm2, %xmm2 diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll index e248aafab52557a8e1591735b107365d796b023c..7a2dcd1c8ca8c9904b3295fd7e6e405410862a0e 100644 --- a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll +++ b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll @@ -941,9 +941,9 @@ define <16 x i8> @splatvar_modulo_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwi ; ; XOPAVX1-LABEL: splatvar_modulo_shift_v16i8: ; XOPAVX1: # %bb.0: -; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm1 ; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm0 ; XOPAVX1-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll index 77f5f2660af7ed1436a44ead3ed347287d23936e..5fe661c7e77780de061207b1706a512938937690 100644 --- a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll +++ b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll @@ -1037,9 +1037,9 @@ define <32 x i8> @splatvar_modulo_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwi ; ; XOPAVX1-LABEL: splatvar_modulo_shift_v32i8: ; XOPAVX1: # %bb.0: -; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm1 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 ; XOPAVX1-NEXT: vpshlb %xmm1, %xmm2, %xmm2 diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll index deb1514e42c4a20798019115f9f89997ad8386ad..12f971fb83b56fae47afcc23c15c0ce2983c2419 100644 --- a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll +++ b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll @@ -848,9 +848,9 @@ define <16 x i8> @splatvar_modulo_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwi ; ; XOPAVX1-LABEL: splatvar_modulo_shift_v16i8: ; XOPAVX1: # %bb.0: -; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm0 ; XOPAVX1-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll index c355eeaa42b669644b748ba3c6e1c27bc38edd65..76944994c87d17994a4d8c3b3d7358b61c1adc5d 100644 --- a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll +++ b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll @@ -962,9 +962,9 @@ define <32 x i8> @splatvar_modulo_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwi ; ; XOPAVX1-LABEL: splatvar_modulo_shift_v32i8: ; XOPAVX1: # %bb.0: -; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 ; XOPAVX1-NEXT: vpshlb %xmm1, %xmm2, %xmm2 ; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm0 diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll index 1b6d2a2c6298eda9ee442e53029dd1bfa16681df..be4253b6d5d1096355d336c8a1cbb95fbd4896b4 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll @@ -3225,7 +3225,7 @@ define void @PR43024() { ; AVX-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}+4(%rip), %xmm0, %xmm0 ; AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1 ; AVX-NEXT: vaddss %xmm1, %xmm0, %xmm0 -; AVX-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}+12(%rip), %xmm0, %xmm0 +; AVX-NEXT: vaddss %xmm1, %xmm0, %xmm0 ; AVX-NEXT: vmovss %xmm0, (%rax) ; AVX-NEXT: retq store <4 x float> , ptr undef, align 16 diff --git a/llvm/test/CodeGen/X86/vector-shuffle-v192.ll b/llvm/test/CodeGen/X86/vector-shuffle-v192.ll index 7159edc2bbdf4cd86873fe8e4c5fe5dabaf1ed7c..75c26f38eb9ed7c54107ca3c33194a2a01d89d95 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-v192.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-v192.ll @@ -214,23 +214,21 @@ define <64 x i8> @f2(ptr %p0) { ; AVX512F-NEXT: vmovdqa 128(%rdi), %ymm4 ; AVX512F-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,3,5,9,11,15,17,21,23,27,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero ; AVX512F-NEXT: vinserti64x4 $1, %ymm4, %zmm2, %zmm2 -; AVX512F-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255] -; AVX512F-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3] -; AVX512F-NEXT: vpternlogq $234, %zmm2, %zmm0, %zmm4 +; AVX512F-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2 ; AVX512F-NEXT: vmovdqa 96(%rdi), %xmm0 ; AVX512F-NEXT: vpshufb %xmm5, %xmm0, %xmm0 -; AVX512F-NEXT: vmovdqa 112(%rdi), %xmm2 -; AVX512F-NEXT: vpshufb %xmm6, %xmm2, %xmm2 -; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0 +; AVX512F-NEXT: vmovdqa 112(%rdi), %xmm4 +; AVX512F-NEXT: vpshufb %xmm6, %xmm4, %xmm4 +; AVX512F-NEXT: vpor %xmm0, %xmm4, %xmm0 ; AVX512F-NEXT: vinserti32x4 $2, %xmm0, %zmm0, %zmm0 -; AVX512F-NEXT: vmovdqa 80(%rdi), %xmm2 -; AVX512F-NEXT: vpshufb %xmm1, %xmm2, %xmm1 -; AVX512F-NEXT: vmovdqa 64(%rdi), %xmm2 -; AVX512F-NEXT: vpshufb %xmm3, %xmm2, %xmm2 -; AVX512F-NEXT: vpor %xmm1, %xmm2, %xmm1 +; AVX512F-NEXT: vmovdqa 80(%rdi), %xmm4 +; AVX512F-NEXT: vpshufb %xmm1, %xmm4, %xmm1 +; AVX512F-NEXT: vmovdqa 64(%rdi), %xmm4 +; AVX512F-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX512F-NEXT: vpor %xmm1, %xmm3, %xmm1 ; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 ; AVX512F-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7] -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-NEXT: retq ; ; AVX512BW-LABEL: f2: @@ -253,11 +251,9 @@ define <64 x i8> @f2(ptr %p0) { ; AVX512BW-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0 ; AVX512BW-NEXT: vmovdqa 32(%rdi), %ymm2 ; AVX512BW-NEXT: vinserti64x4 $1, 128(%rdi), %zmm2, %zmm2 -; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = -; AVX512BW-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3] ; AVX512BW-NEXT: movabsq $8998403163813888, %rax # imm = 0x1FF800001FF800 ; AVX512BW-NEXT: kmovq %rax, %k1 -; AVX512BW-NEXT: vpshufb %zmm4, %zmm2, %zmm0 {%k1} +; AVX512BW-NEXT: vpshufb {{.*#+}} zmm0 {%k1} = zmm2[u,u,u,u,u,u,u,u,u,u,u,3,5,9,11,15,17,21,23,27,29,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,35,37,41,43,47,49,53,55,59,61,u,u,u,u,u,u,u,u,u,u,u] ; AVX512BW-NEXT: vmovdqa 96(%rdi), %xmm2 ; AVX512BW-NEXT: vpshufb %xmm5, %xmm2, %xmm2 ; AVX512BW-NEXT: vmovdqa 112(%rdi), %xmm4 @@ -501,23 +497,21 @@ define <64 x i8> @f4(ptr %p0) { ; AVX512F-NEXT: vmovdqa 128(%rdi), %ymm4 ; AVX512F-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,2,4,8,10,14,16,20,22,26,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero ; AVX512F-NEXT: vinserti64x4 $1, %ymm4, %zmm2, %zmm2 -; AVX512F-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255] -; AVX512F-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3] -; AVX512F-NEXT: vpternlogq $234, %zmm2, %zmm0, %zmm4 +; AVX512F-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2 ; AVX512F-NEXT: vmovdqa 96(%rdi), %xmm0 ; AVX512F-NEXT: vpshufb %xmm5, %xmm0, %xmm0 -; AVX512F-NEXT: vmovdqa 112(%rdi), %xmm2 -; AVX512F-NEXT: vpshufb %xmm6, %xmm2, %xmm2 -; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0 +; AVX512F-NEXT: vmovdqa 112(%rdi), %xmm4 +; AVX512F-NEXT: vpshufb %xmm6, %xmm4, %xmm4 +; AVX512F-NEXT: vpor %xmm0, %xmm4, %xmm0 ; AVX512F-NEXT: vinserti32x4 $2, %xmm0, %zmm0, %zmm0 -; AVX512F-NEXT: vmovdqa 80(%rdi), %xmm2 -; AVX512F-NEXT: vpshufb %xmm1, %xmm2, %xmm1 -; AVX512F-NEXT: vmovdqa 64(%rdi), %xmm2 -; AVX512F-NEXT: vpshufb %xmm3, %xmm2, %xmm2 -; AVX512F-NEXT: vpor %xmm1, %xmm2, %xmm1 +; AVX512F-NEXT: vmovdqa 80(%rdi), %xmm4 +; AVX512F-NEXT: vpshufb %xmm1, %xmm4, %xmm1 +; AVX512F-NEXT: vmovdqa 64(%rdi), %xmm4 +; AVX512F-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX512F-NEXT: vpor %xmm1, %xmm3, %xmm1 ; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 ; AVX512F-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7] -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-NEXT: retq ; ; AVX512BW-LABEL: f4: @@ -540,11 +534,9 @@ define <64 x i8> @f4(ptr %p0) { ; AVX512BW-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0 ; AVX512BW-NEXT: vmovdqa 32(%rdi), %ymm2 ; AVX512BW-NEXT: vinserti64x4 $1, 128(%rdi), %zmm2, %zmm2 -; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = -; AVX512BW-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3] ; AVX512BW-NEXT: movabsq $8998403163813888, %rax # imm = 0x1FF800001FF800 ; AVX512BW-NEXT: kmovq %rax, %k1 -; AVX512BW-NEXT: vpshufb %zmm4, %zmm2, %zmm0 {%k1} +; AVX512BW-NEXT: vpshufb {{.*#+}} zmm0 {%k1} = zmm2[u,u,u,u,u,u,u,u,u,u,u,2,4,8,10,14,16,20,22,26,28,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,34,36,40,42,46,48,52,54,58,60,u,u,u,u,u,u,u,u,u,u,u] ; AVX512BW-NEXT: vmovdqa 96(%rdi), %xmm2 ; AVX512BW-NEXT: vpshufb %xmm5, %xmm2, %xmm2 ; AVX512BW-NEXT: vmovdqa 112(%rdi), %xmm4 diff --git a/llvm/test/CodeGen/X86/x86-interleaved-access.ll b/llvm/test/CodeGen/X86/x86-interleaved-access.ll index 216332943993e374146b8bab74cf1202e85e374b..4f20a9d5db6ac5cab865184e22b3576cba97c3fc 100644 --- a/llvm/test/CodeGen/X86/x86-interleaved-access.ll +++ b/llvm/test/CodeGen/X86/x86-interleaved-access.ll @@ -1054,8 +1054,8 @@ define void @interleaved_store_vf32_i8_stride3(<32 x i8> %a, <32 x i8> %b, <32 x ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm3 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] ; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3] -; AVX512-NEXT: vbroadcasti64x4 {{.*#+}} zmm2 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] -; AVX512-NEXT: # zmm2 = mem[0,1,2,3,0,1,2,3] +; AVX512-NEXT: vbroadcasti32x4 {{.*#+}} zmm2 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] +; AVX512-NEXT: # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3] ; AVX512-NEXT: vpshufb %ymm2, %ymm0, %ymm0 ; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1 ; AVX512-NEXT: vpshufb %zmm2, %zmm1, %zmm1 @@ -1252,8 +1252,8 @@ define void @interleaved_store_vf64_i8_stride3(<64 x i8> %a, <64 x i8> %b, <64 x ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] ; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3] ; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm2 -; AVX512-NEXT: vbroadcasti64x4 {{.*#+}} zmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] -; AVX512-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3] +; AVX512-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] +; AVX512-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3] ; AVX512-NEXT: vpshufb %zmm3, %zmm2, %zmm2 ; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm4 ; AVX512-NEXT: vpshufb %zmm3, %zmm4, %zmm4 diff --git a/llvm/test/CodeGen/X86/zero-remat.ll b/llvm/test/CodeGen/X86/zero-remat.ll index 60bb2c420cda4d34d39d58eee1011aa23c751f4c..000e0d14b711f644faeff45558f9b8ab6ba6a807 100644 --- a/llvm/test/CodeGen/X86/zero-remat.ll +++ b/llvm/test/CodeGen/X86/zero-remat.ll @@ -19,11 +19,12 @@ define double @foo() nounwind { ; ; CHECK-32-LABEL: foo: ; CHECK-32: # %bb.0: -; CHECK-32-NEXT: pushl $0 -; CHECK-32-NEXT: pushl $0 +; CHECK-32-NEXT: subl $8, %esp +; CHECK-32-NEXT: fldz +; CHECK-32-NEXT: fstpl (%esp) ; CHECK-32-NEXT: calll bar@PLT -; CHECK-32-NEXT: addl $8, %esp ; CHECK-32-NEXT: fldz +; CHECK-32-NEXT: addl $8, %esp ; CHECK-32-NEXT: retl call void @bar(double 0.0) ret double 0.0 diff --git a/llvm/test/DebugInfo/assignment-tracking/X86/large-type.ll b/llvm/test/DebugInfo/assignment-tracking/X86/large-type.ll new file mode 100644 index 0000000000000000000000000000000000000000..cebbc162fcb3a8a5fbd80863f9a33cb9d61c81cb --- /dev/null +++ b/llvm/test/DebugInfo/assignment-tracking/X86/large-type.ll @@ -0,0 +1,58 @@ +; RUN: llc %s -stop-after=finalize-isel -o - \ +; RUN: | FileCheck %s --implicit-check-not=DBG_ + +;; Based on optimized IR from C source: +;; int main () { +;; char a1[__INT_MAX__]; +;; a1[__INT_MAX__ - 1] = 5; +;; return a1[__INT_MAX__ - 1]; +;; } +;; +;; Check extremely large types don't cause a crash. +; CHECK: DBG_VALUE 5, $noreg, ![[#]], !DIExpression(DW_OP_LLVM_fragment, 4294967280, 8) +; CHECK: DBG_VALUE 6, $noreg, ![[#]], !DIExpression(DW_OP_LLVM_fragment, 0, 8) +; CHECK: DBG_VALUE 7, $noreg, ![[#]], !DIExpression(DW_OP_LLVM_fragment, 0, 8) + +define dso_local i32 @main() local_unnamed_addr !dbg !10 { +entry: +;; FIXME: SROA currently creates incorrect fragments if bit_offset > max(u32), +;; with and without assignment-tracking. + tail call void @llvm.dbg.value(metadata i8 5, metadata !15, metadata !DIExpression(DW_OP_LLVM_fragment, 4294967280, 8)), !dbg !20 +;; These two were inserted by hand. + tail call void @llvm.dbg.value(metadata i8 6, metadata !22, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !20 + tail call void @llvm.dbg.value(metadata i8 7, metadata !23, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !20 + ret i32 5, !dbg !21 +} + +declare void @llvm.dbg.value(metadata, metadata, metadata) + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8} +!llvm.ident = !{!9} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 18.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "test.c", directory: "/") +!2 = !{i32 7, !"Dwarf Version", i32 5} +!3 = !{i32 2, !"Debug Info Version", i32 3} +!4 = !{i32 1, !"wchar_size", i32 4} +!5 = !{i32 8, !"PIC Level", i32 2} +!6 = !{i32 7, !"PIE Level", i32 2} +!7 = !{i32 7, !"uwtable", i32 2} +!8 = !{i32 7, !"debug-info-assignment-tracking", i1 true} +!9 = !{!"clang version 18.0.0"} +!10 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 3, type: !11, scopeLine: 4, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !14) +!11 = !DISubroutineType(types: !12) +!12 = !{!13} +!13 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!14 = !{!15} +!15 = !DILocalVariable(name: "a1", scope: !10, file: !1, line: 5, type: !16) +!16 = !DICompositeType(tag: DW_TAG_array_type, baseType: !17, size: 17179869176, elements: !18) +!17 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char) +!18 = !{!19} +!19 = !DISubrange(count: 2147483647) +!20 = !DILocation(line: 0, scope: !10) +!21 = !DILocation(line: 7, column: 3, scope: !10) +!22 = !DILocalVariable(name: "a2", scope: !10, file: !1, line: 5, type: !16) +!23 = !DILocalVariable(name: "a3", scope: !10, file: !1, line: 5, type: !16) +!24 = !DICompositeType(tag: DW_TAG_array_type, baseType: !17, size: 4294967232, elements: !18) +!25 = !DICompositeType(tag: DW_TAG_array_type, baseType: !17, size: 4294967233, elements: !18) diff --git a/llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1_elf.ll b/llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1_elf.ll index 659dbe109ec691a6ce77deec87924ef705c6ff9c..0cdc5e7de844dbbe977db9a04f2b3f31a4102f4e 100644 --- a/llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1_elf.ll +++ b/llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1_elf.ll @@ -1,5 +1,4 @@ ; ModuleID = 'argc_sub1.c' -target triple = "x86_64-unknown-unknown-elf" define i32 @sub1(i32) !dbg !8 { call void @llvm.dbg.value(metadata i32 %0, metadata !13, metadata !DIExpression()), !dbg !14 diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_code_model.ll b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_code_model.ll new file mode 100644 index 0000000000000000000000000000000000000000..c1e7694d4cd536a264a22f2015f684a0bfb74457 --- /dev/null +++ b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_code_model.ll @@ -0,0 +1,10 @@ +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes=asan -S | FileCheck %s --check-prefix=LARGE +; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -passes=asan -S | FileCheck %s --check-prefix=NORMAL +; RUN: opt < %s -mtriple=x86_64-pc-windows -passes=asan -S | FileCheck %s --check-prefix=NORMAL + +; check that asan globals metadata are emitted to a large section for x86-64 ELF + +; LARGE: @__asan_global_global = {{.*}}global {{.*}}, code_model "large" +; NORMAL-NOT: code_model "large" + +@global = global i32 0, align 4 diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll b/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll index 699b8287d358ad65f0c58d8386bf00ddbc8c7f15..74f8fc9997d40703b1c2026aba03a24a3e2dacc8 100644 --- a/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll +++ b/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll @@ -102,8 +102,8 @@ target triple = "x86_64-unknown-linux-gnu" ;; Don't place the instrumented globals in a comdat when the unique module ID is empty. ; NOMODULEID: @.str = internal constant { [4 x i8], [28 x i8] } { [4 x i8] c"str\00", [28 x i8] zeroinitializer }, align 32 ; NOMODULEID: @_ZL3buf = internal global { [4 x i8], [28 x i8] } zeroinitializer, align 32 -; NOMODULEID: @__asan_global_.str = private global {{.*}}, section "asan_globals", !associated !0 -; NOMODULEID: @__asan_global__ZL3buf = private global {{.*}}, section "asan_globals", !associated !1 +; NOMODULEID: @__asan_global_.str = private global {{.*}}, section "asan_globals"{{.*}}, !associated !0 +; NOMODULEID: @__asan_global__ZL3buf = private global {{.*}}, section "asan_globals"{{.*}}, !associated !1 ; NOMODULEID: @llvm.compiler.used = appending global [4 x ptr] [ptr @.str, ptr @_ZL3buf, ptr @__asan_global_.str, ptr @__asan_global__ZL3buf] ; NOMODULEID: define internal void @asan.module_ctor() #[[#]] comdat { diff --git a/llvm/test/Instrumentation/InstrProfiling/icall-comdat.ll b/llvm/test/Instrumentation/InstrProfiling/icall-comdat.ll index c90f37c82089572467535b2cb71fb51c52b2dfe6..9fbff456ff50b51a44b41272f44cdaa54f1f450e 100644 --- a/llvm/test/Instrumentation/InstrProfiling/icall-comdat.ll +++ b/llvm/test/Instrumentation/InstrProfiling/icall-comdat.ll @@ -1,5 +1,4 @@ ;; Check that static counters are allocated for value profiler - ; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes=instrprof -vp-static-alloc=true -S | FileCheck %s --check-prefix=STATIC ; RUN: opt < %s -mtriple=powerpc-unknown-linux -passes=instrprof -vp-static-alloc=true -S | FileCheck %s --check-prefix=STATIC ; RUN: opt < %s -mtriple=sparc-unknown-linux -passes=instrprof -vp-static-alloc=true -S | FileCheck %s --check-prefix=STATIC @@ -16,6 +15,10 @@ ; RUN: opt %s -mtriple=powerpc64-ibm-aix -passes=instrprof -S | FileCheck %s --check-prefix=ALIGN ; RUN: opt %s -mtriple=x86_64-unknown-linux -passes=instrprof -S | FileCheck %s --check-prefix=ALIGN +;; Check that globals have the proper code model. +; RUN: opt %s -mtriple=x86_64-unknown-linux -passes=instrprof -S | FileCheck %s --check-prefixes=CODEMODEL,CODEMODEL-X8664 +; RUN: opt %s -mtriple=powerpc-unknown-linux -passes=instrprof -S | FileCheck %s --check-prefixes=CODEMODEL,CODEMODEL-PPC + @__profn_foo = private constant [3 x i8] c"foo" @__profn_bar = private constant [3 x i8] c"bar" @@ -46,8 +49,8 @@ declare void @llvm.instrprof.value.profile(ptr, i64, i64, i32, i32) #0 attributes #0 = { nounwind } -; STATIC: @__profvp_foo = private global [1 x i64] zeroinitializer, section "{{[^"]+}}", comdat($__profc_foo) -; STATIC: @__profvp_bar = private global [1 x i64] zeroinitializer, section "{{[^"]+}}", comdat($__profc_bar) +; STATIC: @__profvp_foo = private global [1 x i64] zeroinitializer, section "{{[^"]+}}",{{.*}} comdat($__profc_foo) +; STATIC: @__profvp_bar = private global [1 x i64] zeroinitializer, section "{{[^"]+}}",{{.*}} comdat($__profc_bar) ; STATIC: @__llvm_prf_vnodes ; DYN-NOT: @__profvp_foo @@ -73,5 +76,26 @@ attributes #0 = { nounwind } ; ALIGN: @__profc_bar = private global {{.*}} section "__llvm_prf_cnts",{{.*}} align 8 ; ALIGN: @__profvp_bar = private global {{.*}} section "__llvm_prf_vals",{{.*}} align 8 ; ALIGN: @__profd_bar = private global {{.*}} section "__llvm_prf_data",{{.*}} align 8 -; ALIGN: @__llvm_prf_vnodes = private global {{.*}} section "__llvm_prf_vnds", align 8 -; ALIGN: @__llvm_prf_nm = private constant {{.*}} section "__llvm_prf_names", align 1 +; ALIGN: @__llvm_prf_vnodes = private global {{.*}} section "__llvm_prf_vnds",{{.*}} align 8 +; ALIGN: @__llvm_prf_nm = private constant {{.*}} section "__llvm_prf_names",{{.*}} align 1 + +; CODEMODEL: @__profc_foo = +; CODEMODEL-NOT: code_model "large" +; CODEMODEL: @__profvp_foo = +; CODEMODEL-X8664-SAME: code_model "large" +; CODEMODEL-PPC-NOT: code_model +; CODEMODEL: @__profd_foo = +; CODEMODEL-NOT: code_model "large" +; CODEMODEL: @__profc_bar = +; CODEMODEL-NOT: code_model "large" +; CODEMODEL: @__profvp_bar = +; CODEMODEL-X8664-SAME: code_model "large" +; CODEMODEL-PPC-NOT: code_model +; CODEMODEL: @__profd_bar = +; CODEMODEL-NOT: code_model "large" +; CODEMODEL: @__llvm_prf_vnodes = +; CODEMODEL-X8664-SAME: code_model "large" +; CODEMODEL-PPC-NOT: code_model +; CODEMODEL: @__llvm_prf_nm = +; CODEMODEL-X8664-SAME: code_model "large" +; CODEMODEL-PPC-NOT: code_model diff --git a/llvm/test/Instrumentation/InstrProfiling/platform.ll b/llvm/test/Instrumentation/InstrProfiling/platform.ll index 3c3712240d1e8eeb6bf74c33fa859c447876abca..81912f3c0b6a9497a70aef487719862de5987fd0 100644 --- a/llvm/test/Instrumentation/InstrProfiling/platform.ll +++ b/llvm/test/Instrumentation/InstrProfiling/platform.ll @@ -25,7 +25,7 @@ ; WINDOWS: @__profd_foo = private global {{.*}}, section ".lprfd$M", align 8 ; AIX: @__profd_foo = private {{.*}}, section "__llvm_prf_data", align 8 -; ELF: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}__llvm_prf_names", align 1 +; ELF: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}__llvm_prf_names"{{.*}}, align 1 ; WINDOWS: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}lprfn$M", align 1 ; AIX: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}__llvm_prf_names", align 1 diff --git a/llvm/test/MC/AArch64/elf-reloc-ptrauth.s b/llvm/test/MC/AArch64/elf-reloc-ptrauth.s index 1ce008117ac30207ef8d833357d0ab1f7d5b3cf2..3bd8f5c19932ee9301aa96e80462331e2331125b 100644 --- a/llvm/test/MC/AArch64/elf-reloc-ptrauth.s +++ b/llvm/test/MC/AArch64/elf-reloc-ptrauth.s @@ -5,14 +5,14 @@ // RELOC: Relocation section '.rela.test' at offset 0x230 contains 8 entries: // RELOC-NEXT: Offset Info Type Symbol's Value Symbol's Name + Addend -// RELOC-NEXT: 0000000000000000 000000010000e100 R_AARCH64_AUTH_ABS64 0000000000000000 .helper + 0 -// RELOC-NEXT: 0000000000000010 000000080000e100 R_AARCH64_AUTH_ABS64 0000000000000000 _g1 + 0 -// RELOC-NEXT: 0000000000000020 000000090000e100 R_AARCH64_AUTH_ABS64 0000000000000000 _g2 + 0 -// RELOC-NEXT: 0000000000000030 0000000a0000e100 R_AARCH64_AUTH_ABS64 0000000000000000 _g3 + 0 -// RELOC-NEXT: 0000000000000040 0000000b0000e100 R_AARCH64_AUTH_ABS64 0000000000000000 _g4 + 7 -// RELOC-NEXT: 0000000000000050 0000000c0000e100 R_AARCH64_AUTH_ABS64 0000000000000000 _g5 - 3 -// RELOC-NEXT: 0000000000000060 000000020000e100 R_AARCH64_AUTH_ABS64 0000000000000000 _g 6 + 0 -// RELOC-NEXT: 0000000000000070 0000000d0000e100 R_AARCH64_AUTH_ABS64 0000000000000000 _g 7 + 7 +// RELOC-NEXT: 0000000000000000 0000000100000244 R_AARCH64_AUTH_ABS64 0000000000000000 .helper + 0 +// RELOC-NEXT: 0000000000000010 0000000800000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g1 + 0 +// RELOC-NEXT: 0000000000000020 0000000900000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g2 + 0 +// RELOC-NEXT: 0000000000000030 0000000a00000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g3 + 0 +// RELOC-NEXT: 0000000000000040 0000000b00000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g4 + 7 +// RELOC-NEXT: 0000000000000050 0000000c00000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g5 - 3 +// RELOC-NEXT: 0000000000000060 0000000200000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g 6 + 0 +// RELOC-NEXT: 0000000000000070 0000000d00000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g 7 + 7 // RELOC: Hex dump of section '.test': // VVVVVVVV addend, not needed for rela diff --git a/llvm/test/MC/AMDGPU/exp.s b/llvm/test/MC/AMDGPU/exp.s index a2fa09b71496c675c6febd8c306ece32d3e24623..d04e15202a586c0ab36ac7278fa7ec7a13137a28 100644 --- a/llvm/test/MC/AMDGPU/exp.s +++ b/llvm/test/MC/AMDGPU/exp.s @@ -3,103 +3,124 @@ // RUN: llvm-mc -triple=amdgcn -mcpu=gfx900 -show-encoding %s | FileCheck -check-prefix=GFX89 %s // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1010 -show-encoding %s | FileCheck -check-prefix=GFX10 %s // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck -check-prefix=GFX10 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GFX12 %s exp mrt0 off, off, off, off // SI: exp mrt0 off, off, off, off ; encoding: [0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x00] // GFX89: exp mrt0 off, off, off, off ; encoding: [0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x00] // GFX10: exp mrt0 off, off, off, off ; encoding: [0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x00] +// GFX12: export mrt0 off, off, off, off ; encoding: [0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x00] exp mrt0 off, off, off, off done // SI: exp mrt0 off, off, off, off done ; encoding: [0x00,0x08,0x00,0xf8,0x00,0x00,0x00,0x00] // GFX89: exp mrt0 off, off, off, off done ; encoding: [0x00,0x08,0x00,0xc4,0x00,0x00,0x00,0x00] // GFX10: exp mrt0 off, off, off, off done ; encoding: [0x00,0x08,0x00,0xf8,0x00,0x00,0x00,0x00] +// GFX12: export mrt0 off, off, off, off done ; encoding: [0x00,0x08,0x00,0xf8,0x00,0x00,0x00,0x00] exp mrt0 v4, off, off, off done // SI: exp mrt0 v4, off, off, off done ; encoding: [0x01,0x08,0x00,0xf8,0x04,0x00,0x00,0x00] // GFX89: exp mrt0 v4, off, off, off done ; encoding: [0x01,0x08,0x00,0xc4,0x04,0x00,0x00,0x00] // GFX10: exp mrt0 v4, off, off, off done ; encoding: [0x01,0x08,0x00,0xf8,0x04,0x00,0x00,0x00] +// GFX12: export mrt0 v4, off, off, off done ; encoding: [0x01,0x08,0x00,0xf8,0x04,0x00,0x00,0x00] exp mrt0 off, v3, off, off done // SI: exp mrt0 off, v3, off, off done ; encoding: [0x02,0x08,0x00,0xf8,0x00,0x03,0x00,0x00] // GFX89: exp mrt0 off, v3, off, off done ; encoding: [0x02,0x08,0x00,0xc4,0x00,0x03,0x00,0x00] // GFX10: exp mrt0 off, v3, off, off done ; encoding: [0x02,0x08,0x00,0xf8,0x00,0x03,0x00,0x00] +// GFX12: export mrt0 off, v3, off, off done ; encoding: [0x02,0x08,0x00,0xf8,0x00,0x03,0x00,0x00] exp mrt0 off, off, v2, off done // SI: exp mrt0 off, off, v2, off done ; encoding: [0x04,0x08,0x00,0xf8,0x00,0x00,0x02,0x00] // GFX89: exp mrt0 off, off, v2, off done ; encoding: [0x04,0x08,0x00,0xc4,0x00,0x00,0x02,0x00] // GFX10: exp mrt0 off, off, v2, off done ; encoding: [0x04,0x08,0x00,0xf8,0x00,0x00,0x02,0x00] +// GFX12: export mrt0 off, off, v2, off done ; encoding: [0x04,0x08,0x00,0xf8,0x00,0x00,0x02,0x00] exp mrt0 off, off, off, v1 done // SI: exp mrt0 off, off, off, v1 done ; encoding: [0x08,0x08,0x00,0xf8,0x00,0x00,0x00,0x01] // GFX89: exp mrt0 off, off, off, v1 done ; encoding: [0x08,0x08,0x00,0xc4,0x00,0x00,0x00,0x01] // GFX10: exp mrt0 off, off, off, v1 done ; encoding: [0x08,0x08,0x00,0xf8,0x00,0x00,0x00,0x01] +// GFX12: export mrt0 off, off, off, v1 done ; encoding: [0x08,0x08,0x00,0xf8,0x00,0x00,0x00,0x01] exp mrt0 v4, v3, off, off done // SI: exp mrt0 v4, v3, off, off done ; encoding: [0x03,0x08,0x00,0xf8,0x04,0x03,0x00,0x00] // GFX89: exp mrt0 v4, v3, off, off done ; encoding: [0x03,0x08,0x00,0xc4,0x04,0x03,0x00,0x00] // GFX10: exp mrt0 v4, v3, off, off done ; encoding: [0x03,0x08,0x00,0xf8,0x04,0x03,0x00,0x00] +// GFX12: export mrt0 v4, v3, off, off done ; encoding: [0x03,0x08,0x00,0xf8,0x04,0x03,0x00,0x00] exp mrt0 v4, off, v2, off done // SI: exp mrt0 v4, off, v2, off done ; encoding: [0x05,0x08,0x00,0xf8,0x04,0x00,0x02,0x00] // GFX89: exp mrt0 v4, off, v2, off done ; encoding: [0x05,0x08,0x00,0xc4,0x04,0x00,0x02,0x00] // GFX10: exp mrt0 v4, off, v2, off done ; encoding: [0x05,0x08,0x00,0xf8,0x04,0x00,0x02,0x00] +// GFX12: export mrt0 v4, off, v2, off done ; encoding: [0x05,0x08,0x00,0xf8,0x04,0x00,0x02,0x00] exp mrt0 v4, off, off, v1 // SI: exp mrt0 v4, off, off, v1 ; encoding: [0x09,0x00,0x00,0xf8,0x04,0x00,0x00,0x01] // GFX89: exp mrt0 v4, off, off, v1 ; encoding: [0x09,0x00,0x00,0xc4,0x04,0x00,0x00,0x01] // GFX10: exp mrt0 v4, off, off, v1 ; encoding: [0x09,0x00,0x00,0xf8,0x04,0x00,0x00,0x01] +// GFX12: export mrt0 v4, off, off, v1 ; encoding: [0x09,0x00,0x00,0xf8,0x04,0x00,0x00,0x01] exp mrt0 v4, off, off, v1 done // SI: exp mrt0 v4, off, off, v1 done ; encoding: [0x09,0x08,0x00,0xf8,0x04,0x00,0x00,0x01] // GFX89: exp mrt0 v4, off, off, v1 done ; encoding: [0x09,0x08,0x00,0xc4,0x04,0x00,0x00,0x01] // GFX10: exp mrt0 v4, off, off, v1 done ; encoding: [0x09,0x08,0x00,0xf8,0x04,0x00,0x00,0x01] +// GFX12: export mrt0 v4, off, off, v1 done ; encoding: [0x09,0x08,0x00,0xf8,0x04,0x00,0x00,0x01] exp mrt0 v4, v3, v2, v1 // SI: exp mrt0 v4, v3, v2, v1 ; encoding: [0x0f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp mrt0 v4, v3, v2, v1 ; encoding: [0x0f,0x00,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp mrt0 v4, v3, v2, v1 ; encoding: [0x0f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export mrt0 v4, v3, v2, v1 ; encoding: [0x0f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] exp mrt0 v4, v3, v2, v1 done // SI: exp mrt0 v4, v3, v2, v1 done ; encoding: [0x0f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp mrt0 v4, v3, v2, v1 done ; encoding: [0x0f,0x08,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp mrt0 v4, v3, v2, v1 done ; encoding: [0x0f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export mrt0 v4, v3, v2, v1 done ; encoding: [0x0f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] exp mrt7 v1, v1, v1, v1 // SI: exp mrt7 v1, v1, v1, v1 ; encoding: [0x7f,0x00,0x00,0xf8,0x01,0x01,0x01,0x01] // GFX89: exp mrt7 v1, v1, v1, v1 ; encoding: [0x7f,0x00,0x00,0xc4,0x01,0x01,0x01,0x01] // GFX10: exp mrt7 v1, v1, v1, v1 ; encoding: [0x7f,0x00,0x00,0xf8,0x01,0x01,0x01,0x01] +// GFX12: export mrt7 v1, v1, v1, v1 ; encoding: [0x7f,0x00,0x00,0xf8,0x01,0x01,0x01,0x01] exp mrt7 v1, v1, v1, v1 done // SI: exp mrt7 v1, v1, v1, v1 done ; encoding: [0x7f,0x08,0x00,0xf8,0x01,0x01,0x01,0x01] // GFX89: exp mrt7 v1, v1, v1, v1 done ; encoding: [0x7f,0x08,0x00,0xc4,0x01,0x01,0x01,0x01] // GFX10: exp mrt7 v1, v1, v1, v1 done ; encoding: [0x7f,0x08,0x00,0xf8,0x01,0x01,0x01,0x01] +// GFX12: export mrt7 v1, v1, v1, v1 done ; encoding: [0x7f,0x08,0x00,0xf8,0x01,0x01,0x01,0x01] exp mrtz v4, v3, v2, v1 // SI: exp mrtz v4, v3, v2, v1 ; encoding: [0x8f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp mrtz v4, v3, v2, v1 ; encoding: [0x8f,0x00,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp mrtz v4, v3, v2, v1 ; encoding: [0x8f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export mrtz v4, v3, v2, v1 ; encoding: [0x8f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] exp mrtz v4, v3, v2, v1 done // SI: exp mrtz v4, v3, v2, v1 done ; encoding: [0x8f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp mrtz v4, v3, v2, v1 done ; encoding: [0x8f,0x08,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp mrtz v4, v3, v2, v1 done ; encoding: [0x8f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export mrtz v4, v3, v2, v1 done ; encoding: [0x8f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] exp pos0 v4, v3, v2, v1 // SI: exp pos0 v4, v3, v2, v1 ; encoding: [0xcf,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp pos0 v4, v3, v2, v1 ; encoding: [0xcf,0x00,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp pos0 v4, v3, v2, v1 ; encoding: [0xcf,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export pos0 v4, v3, v2, v1 ; encoding: [0xcf,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] exp pos0 v4, v3, v2, v1 done // SI: exp pos0 v4, v3, v2, v1 done ; encoding: [0xcf,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp pos0 v4, v3, v2, v1 done ; encoding: [0xcf,0x08,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp pos0 v4, v3, v2, v1 done ; encoding: [0xcf,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export pos0 v4, v3, v2, v1 done ; encoding: [0xcf,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] exp pos3 v4, v3, v2, v1 // SI: exp pos3 v4, v3, v2, v1 ; encoding: [0xff,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp pos3 v4, v3, v2, v1 ; encoding: [0xff,0x00,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp pos3 v4, v3, v2, v1 ; encoding: [0xff,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export pos3 v4, v3, v2, v1 ; encoding: [0xff,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] exp pos3 v4, v3, v2, v1 done // SI: exp pos3 v4, v3, v2, v1 done ; encoding: [0xff,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp pos3 v4, v3, v2, v1 done ; encoding: [0xff,0x08,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp pos3 v4, v3, v2, v1 done ; encoding: [0xff,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export pos3 v4, v3, v2, v1 done ; encoding: [0xff,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s index e2e53776783f30ad3263609b1ea27b08fcaf7eab..fdfbf65c0e3cf302f877ad3667d51f3ca84a7d34 100644 --- a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s +++ b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s @@ -1,277 +1,278 @@ -// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck -check-prefix=GFX11 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck -check-prefix=GCN %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GCN %s v_interp_p10_f32 v0, v1, v2, v3 -// GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f32 v1, v10, v20, v30 -// GFX11: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04] +// GCN: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04] v_interp_p10_f32 v2, v11, v21, v31 -// GFX11: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04] +// GCN: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04] v_interp_p10_f32 v3, v12, v22, v32 -// GFX11: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04] +// GCN: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04] v_interp_p10_f32 v0, v1, v2, v3 clamp -// GFX11: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f32 v0, -v1, v2, v3 -// GFX11: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24] +// GCN: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24] v_interp_p10_f32 v0, v1, -v2, v3 -// GFX11: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44] +// GCN: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44] v_interp_p10_f32 v0, v1, v2, -v3 -// GFX11: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84] +// GCN: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84] v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 -// GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1 -// GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7 -// GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7 -// GFX11: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f32 v0, v1, v2, v3 -// GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f32 v1, v10, v20, v30 -// GFX11: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04] +// GCN: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04] v_interp_p2_f32 v2, v11, v21, v31 -// GFX11: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04] +// GCN: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04] v_interp_p2_f32 v3, v12, v22, v32 -// GFX11: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04] +// GCN: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04] v_interp_p2_f32 v0, v1, v2, v3 clamp -// GFX11: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f32 v0, -v1, v2, v3 -// GFX11: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24] +// GCN: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24] v_interp_p2_f32 v0, v1, -v2, v3 -// GFX11: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44] +// GCN: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44] v_interp_p2_f32 v0, v1, v2, -v3 -// GFX11: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84] +// GCN: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84] v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 -// GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1 -// GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7 -// GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7 -// GFX11: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, -v1, v2, v3 -// GFX11: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24] +// GCN: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24] v_interp_p10_f16_f32 v0, v1, -v2, v3 -// GFX11: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44] +// GCN: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44] v_interp_p10_f16_f32 v0, v1, v2, -v3 -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84] v_interp_p10_f16_f32 v0, v1, v2, v3 clamp -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0xe4] +// GCN: v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0xe4] v_interp_p2_f16_f32 v0, v1, v2, v3 -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, -v1, v2, v3 -// GFX11: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24] +// GCN: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24] v_interp_p2_f16_f32 v0, v1, -v2, v3 -// GFX11: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44] +// GCN: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44] v_interp_p2_f16_f32 v0, v1, v2, -v3 -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84] v_interp_p2_f16_f32 v0, v1, v2, v3 clamp -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0xe4] +// GCN: v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0xe4] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 -// GFX11: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24] +// GCN: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24] v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44] v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0xe4] +// GCN: v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0xe4] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 -// GFX11: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24] +// GCN: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24] v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44] v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0xe4] +// GCN: v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0xe4] diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp_err.s b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp_err.s index 1d6769dbd4d6a087835fdd471eeec48419a6275b..66b04c0d90869c235e972794f70ecdd53861d2ef 100644 --- a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp_err.s +++ b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp_err.s @@ -1,4 +1,5 @@ -// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 %s 2>&1 | FileCheck %s -check-prefix=GFX11-ERR --implicit-check-not=error: --strict-whitespace +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 %s 2>&1 | FileCheck %s -check-prefix=GCN-ERR --implicit-check-not=error: --strict-whitespace +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck %s -check-prefix=GCN-ERR --implicit-check-not=error: --strict-whitespace //===----------------------------------------------------------------------===// // VINTERP src operands must be VGPRs. @@ -6,37 +7,37 @@ //===----------------------------------------------------------------------===// v_interp_p10_f32 v0, s1, v2, v3 -// GFX11-ERR: :[[@LINE-1]]:22: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:22: error: invalid operand for instruction v_interp_p10_f32 v0, v1, s2, v3 -// GFX11-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction v_interp_p10_f32 v0, v1, v2, s3 -// GFX11-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction v_interp_p2_f32 v0, 1, v2, v3 -// GFX11-ERR: :[[@LINE-1]]:21: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:21: error: invalid operand for instruction v_interp_p2_f32 v0, v1, 2, v3 -// GFX11-ERR: :[[@LINE-1]]:25: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:25: error: invalid operand for instruction v_interp_p2_f32 v0, v1, v2, 3 -// GFX11-ERR: :[[@LINE-1]]:29: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:29: error: invalid operand for instruction v_interp_p10_f16_f32 v0, s1, v2, v3 -// GFX11-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction v_interp_p10_f16_f32 v0, v1, s2, v3 -// GFX11-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction v_interp_p10_f16_f32 v0, v1, v2, s3 -// GFX11-ERR: :[[@LINE-1]]:34: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:34: error: invalid operand for instruction v_interp_p2_f16_f32 v0, 1, v2, v3 -// GFX11-ERR: :[[@LINE-1]]:25: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:25: error: invalid operand for instruction v_interp_p2_f16_f32 v0, v1, 2, v3 -// GFX11-ERR: :[[@LINE-1]]:29: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:29: error: invalid operand for instruction v_interp_p2_f16_f32 v0, v1, v2, 3 -// GFX11-ERR: :[[@LINE-1]]:33: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:33: error: invalid operand for instruction diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_exp.s b/llvm/test/MC/AMDGPU/gfx12_asm_exp.s new file mode 100644 index 0000000000000000000000000000000000000000..2390813954a25a132656fe6d86f81de3297c12d9 --- /dev/null +++ b/llvm/test/MC/AMDGPU/gfx12_asm_exp.s @@ -0,0 +1,61 @@ +// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GFX12 %s + +export mrt0 off, off, off, off +// GFX12: export mrt0 off, off, off, off ; encoding: [0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x00] + +export mrt0 off, off, off, off done +// GFX12: export mrt0 off, off, off, off done ; encoding: [0x00,0x08,0x00,0xf8,0x00,0x00,0x00,0x00] + +export mrt0 v4, off, off, off done +// GFX12: export mrt0 v4, off, off, off done ; encoding: [0x01,0x08,0x00,0xf8,0x04,0x00,0x00,0x00] + +export mrt0 off, v3, off, off done +// GFX12: export mrt0 off, v3, off, off done ; encoding: [0x02,0x08,0x00,0xf8,0x00,0x03,0x00,0x00] + +export mrt0 off, off, v2, off done +// GFX12: export mrt0 off, off, v2, off done ; encoding: [0x04,0x08,0x00,0xf8,0x00,0x00,0x02,0x00] + +export mrt0 off, off, off, v1 done +// GFX12: export mrt0 off, off, off, v1 done ; encoding: [0x08,0x08,0x00,0xf8,0x00,0x00,0x00,0x01] + +export mrt0 v4, v3, off, off done +// GFX12: export mrt0 v4, v3, off, off done ; encoding: [0x03,0x08,0x00,0xf8,0x04,0x03,0x00,0x00] + +export mrt0 v4, off, v2, off done +// GFX12: export mrt0 v4, off, v2, off done ; encoding: [0x05,0x08,0x00,0xf8,0x04,0x00,0x02,0x00] + +export mrt0 v4, off, off, v1 +// GFX12: export mrt0 v4, off, off, v1 ; encoding: [0x09,0x00,0x00,0xf8,0x04,0x00,0x00,0x01] + +export mrt0 v4, off, off, v1 done +// GFX12: export mrt0 v4, off, off, v1 done ; encoding: [0x09,0x08,0x00,0xf8,0x04,0x00,0x00,0x01] + +export mrt0 v4, v3, v2, v1 +// GFX12: export mrt0 v4, v3, v2, v1 ; encoding: [0x0f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] + +export mrt0 v4, v3, v2, v1 done +// GFX12: export mrt0 v4, v3, v2, v1 done ; encoding: [0x0f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] + +export mrt7 v1, v1, v1, v1 +// GFX12: export mrt7 v1, v1, v1, v1 ; encoding: [0x7f,0x00,0x00,0xf8,0x01,0x01,0x01,0x01] + +export mrt7 v1, v1, v1, v1 done +// GFX12: export mrt7 v1, v1, v1, v1 done ; encoding: [0x7f,0x08,0x00,0xf8,0x01,0x01,0x01,0x01] + +export mrtz v4, v3, v2, v1 +// GFX12: export mrtz v4, v3, v2, v1 ; encoding: [0x8f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] + +export mrtz v4, v3, v2, v1 done +// GFX12: export mrtz v4, v3, v2, v1 done ; encoding: [0x8f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] + +export pos0 v4, v3, v2, v1 +// GFX12: export pos0 v4, v3, v2, v1 ; encoding: [0xcf,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] + +export pos0 v4, v3, v2, v1 done +// GFX12: export pos0 v4, v3, v2, v1 done ; encoding: [0xcf,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] + +export pos3 v4, v3, v2, v1 +// GFX12: export pos3 v4, v3, v2, v1 ; encoding: [0xff,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] + +export pos3 v4, v3, v2, v1 done +// GFX12: export pos3 v4, v3, v2, v1 done ; encoding: [0xff,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s index 494b8399a26fb52702941c4cda4f3baa1fbf2a7f..db166e8ffc104f7dc059bffcfcf748c9f567b78a 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s @@ -3648,6 +3648,9 @@ s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_SAVE_WAVE) s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TBA) // GFX12: encoding: [0x85,0x4c,0x80,0xbe] +s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_SE_AID_ID) +// GFX12: encoding: [0x86,0x4c,0x80,0xbe] + s_ctz_i32_b32 s5, s1 // GFX12: encoding: [0x01,0x08,0x85,0xbe] diff --git a/llvm/test/MC/AMDGPU/sopp-err.s b/llvm/test/MC/AMDGPU/sopp-err.s index c7f28faa55763befd6eec5003ea766e6b15cc5b8..bd044cb7434025b3604a64d062ee15bb9e911349 100644 --- a/llvm/test/MC/AMDGPU/sopp-err.s +++ b/llvm/test/MC/AMDGPU/sopp-err.s @@ -2,7 +2,8 @@ // RUN: not llvm-mc -triple=amdgcn -mcpu=tahiti %s 2>&1 | FileCheck --check-prefixes=GCN,PREGFX11,SICI,SICIVI --implicit-check-not=error: %s // RUN: not llvm-mc -triple=amdgcn -mcpu=fiji %s 2>&1 | FileCheck --check-prefixes=GCN,PREGFX11,VI,SICIVI --implicit-check-not=error: %s // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1010 %s 2>&1 | FileCheck --check-prefixes=GCN,PREGFX11,GFX10 --implicit-check-not=error: %s -// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 %s 2>&1 | FileCheck --check-prefixes=GCN,GFX11 --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 %s 2>&1 | FileCheck --check-prefixes=GCN,GFX11PLUS,GFX11 --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefixes=GCN,GFX11PLUS,GFX12 --implicit-check-not=error: %s //===----------------------------------------------------------------------===// // sendmsg @@ -22,30 +23,30 @@ s_sendmsg sendmsg(MSG_INTERRUPT, 0, 0) s_sendmsg sendmsg(MSG_GS) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: missing message operation -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS, GS_OP_NOP) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operation id -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS, SYSMSG_OP_ECC_ERR_INTERRUPT) // GCN: :[[@LINE-1]]:{{[0-9]+}}: error: expected an operation name or an absolute expression s_sendmsg sendmsg(MSG_GS, 0) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operation id -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS, -1) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operation id -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS, 4) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operation id -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS, 8) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operation id -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(15, -1) // GCN: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operation id @@ -67,7 +68,7 @@ s_sendmsg sendmsg(MSG_GS, 1 -) s_sendmsg sendmsg(MSG_GS, GS_OP_CUT, 4) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid message stream id -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS, GS_OP_CUT, 1 -) // GCN: :[[@LINE-1]]:{{[0-9]+}}: error: unknown token in expression @@ -86,11 +87,11 @@ s_sendmsg sendmsg(2, 2, 0, 0) s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP, 0) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: message operation does not support streams -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS_DONE, 0, 0) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: message operation does not support streams -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_HS_TESSFACTOR) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU @@ -104,25 +105,27 @@ s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) s_sendmsg sendmsg(MSG_SAVEWAVE) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_STALL_WAVE_GEN) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // VI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX12: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_HALT_WAVES) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // VI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX12: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_ORDERED_PS_DONE) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // VI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU -// GFX11: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_EARLY_PRIM_DEALLOC) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // VI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU -// GFX11: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS_ALLOC_REQ) // VI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU @@ -132,17 +135,17 @@ s_sendmsg sendmsg(MSG_GS_ALLOC_REQ, 0) // VI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // SICI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // GFX10: :[[@LINE-3]]:{{[0-9]+}}: error: message does not support operations -// GFX11: :[[@LINE-4]]:{{[0-9]+}}: error: message does not support operations +// GFX11PLUS: :[[@LINE-4]]:{{[0-9]+}}: error: message does not support operations s_sendmsg sendmsg(MSG_GET_DOORBELL) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // VI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU -// GFX11: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GET_DDID) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // VI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU -// GFX11: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_RTN_GET_DOORBELL) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU @@ -174,6 +177,10 @@ s_sendmsg sendmsg(MSG_RTN_GET_TBA) // VI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // GFX10: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +s_sendmsg sendmsg(MSG_RTN_GET_SE_AID_ID) +// PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU + s_sendmsg sendmsg(-1) // GCN: :[[@LINE-1]]:{{[0-9]+}}: error: invalid message id @@ -252,12 +259,12 @@ s_waitcnt vmcnt(0 s_waitcnt_depctr 65536 // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr -32769 // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_hold_cnt(0) @@ -266,162 +273,162 @@ s_waitcnt_depctr depctr_hold_cnt(0) s_waitcnt_depctr depctr_sa_sdst(-1) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_sa_sdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_sa_sdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_sa_sdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_vdst(-1) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_vdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_sdst(-1) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_sdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_sdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_sdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_ssrc(-1) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_ssrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_ssrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_ssrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_vcc(-1) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_vcc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vcc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vcc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(-1) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_vm_vsrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_vm_vsrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_vm_vsrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_sa_sdst(2) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_sa_sdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_sa_sdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_sa_sdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_vdst(16) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_vdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_sdst(8) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_sdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_sdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_sdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_ssrc(2) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_ssrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_ssrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_ssrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_vcc(2) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_vcc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vcc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vcc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(8) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_vm_vsrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_vm_vsrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_vm_vsrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_(8) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid counter name depctr_vm_ -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid counter name depctr_vm_ +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid counter name depctr_vm_ // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_sa_sdst(0) depctr_sa_sdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_sa_sdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_sa_sdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_sa_sdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_vdst(0) depctr_va_vdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_sdst(0) depctr_va_sdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_va_sdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_sdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_sdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_ssrc(0) depctr_va_ssrc(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_va_ssrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_ssrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_ssrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_vcc(0) depctr_va_vcc(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vcc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vcc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vcc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(0) depctr_vm_vsrc(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_vm_vsrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_vm_vsrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_vm_vsrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_sa_sdst(0) depctr_va_sdst(0) depctr_sa_sdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_sa_sdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_sa_sdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_sa_sdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_ssrc(0) depctr_va_sdst(0) depctr_va_ssrc(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_va_ssrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_ssrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_ssrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_vcc(0) depctr_va_vcc(0) depctr_va_sdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vcc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vcc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vcc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(0) depctr_vm_vsrc(0) depctr_va_sdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_vm_vsrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_vm_vsrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_vm_vsrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_sdst(0) depctr_vm_vsrc 0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected a left parenthesis -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected a left parenthesis +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected a left parenthesis // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_sdst(0) 0depctr_vm_vsrc(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected a counter name -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_sdst(0) depctr_vm_vsrc(x) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected absolute expression -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected absolute expression +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected absolute expression // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_sdst(0) depctr_vm_vsrc(0; & depctr_va_sdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected a closing parenthesis -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected a closing parenthesis +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected a closing parenthesis // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc 0) depctr_vm_vsrc(0) depctr_va_sdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected absolute expression -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected absolute expression +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected absolute expression // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(0) , // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected a counter name -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(0) , & // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected a counter name -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(0) & // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected a counter name -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(0) & & // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected a counter name -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU //===----------------------------------------------------------------------===// diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_exp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_exp.txt new file mode 100644 index 0000000000000000000000000000000000000000..3f173123983b687e2afb2c572ddc078c7731b287 --- /dev/null +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_exp.txt @@ -0,0 +1,13 @@ +# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck %s -check-prefix=GFX12 + +# GFX12: export dual_src_blend0 v4, v3, v2, v1 ; encoding: [0x5f,0x01,0x00,0xf8,0x04,0x03,0x02,0x01] +0x5f,0x01,0x00,0xf8,0x04,0x03,0x02,0x01 + +# GFX12: export dual_src_blend1 v2, v3, off, off ; encoding: [0x63,0x01,0x00,0xf8,0x02,0x03,0x00,0x00] +0x63,0x01,0x00,0xf8,0x02,0x03,0x00,0x00 + +# GFX12: export mrtz v4, v3, v2, v1 row_en ; encoding: [0x8f,0x20,0x00,0xf8,0x04,0x03,0x02,0x01] +0x8f,0x20,0x00,0xf8,0x04,0x03,0x02,0x01 + +# GFX12: export mrtz v4, v3, off, off done row_en ; encoding: [0x83,0x28,0x00,0xf8,0x04,0x03,0x00,0x00] +0x83,0x28,0x00,0xf8,0x04,0x03,0x00,0x00 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt index 7029f090faa4b94a926c79cfe93c7918d79dadb7..c061462339b66c1cad651b3fbc49bd56029d6e4c 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt @@ -3219,6 +3219,9 @@ # GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TBA) ; encoding: [0x85,0x4c,0x80,0xbe] 0x85,0x4c,0x80,0xbe +# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_SE_AID_ID) ; encoding: [0x86,0x4c,0x80,0xbe] +0x86,0x4c,0x80,0xbe + # GFX12: s_setpc_b64 s[0:1] ; encoding: [0x00,0x48,0x80,0xbe] 0x00,0x48,0x80,0xbe diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt new file mode 100644 index 0000000000000000000000000000000000000000..bd1db69625ca5e809265d4623144939495422413 --- /dev/null +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt @@ -0,0 +1,251 @@ +# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s + +# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# Check that unused bits in the encoding are ignored. +# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0xe0,0xcd,0x01,0x05,0x0e,0x1c + +# GFX12: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0{{$}} +0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04 + +# GFX12: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0{{$}} +0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04 + +# GFX12: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0{{$}} +0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04 + +# GFX12: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24 + +# GFX12: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44 + +# GFX12: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84 + +# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} +0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0{{$}} +0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04 + +# GFX12: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0{{$}} +0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04 + +# GFX12: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0{{$}} +0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04 + +# GFX12: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24 + +# GFX12: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44 + +# GFX12: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84 + +# GFX12: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} +0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24 + +# GFX12: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0xe4 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24 + +# GFX12: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0xe4 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0xe4 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0xe4 diff --git a/llvm/test/MC/Disassembler/X86/apx/invpcid.txt b/llvm/test/MC/Disassembler/X86/apx/invpcid.txt new file mode 100644 index 0000000000000000000000000000000000000000..8987332e9c9d9d33f7a9a122275ca7ef68f21f3c --- /dev/null +++ b/llvm/test/MC/Disassembler/X86/apx/invpcid.txt @@ -0,0 +1,6 @@ +# RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s --check-prefixes=ATT +# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL + +# ATT: invpcid 291(%r28,%r29,4), %r19 +# INTEL: invpcid r19, xmmword ptr [r28 + 4*r29 + 291] +0x62,0x8c,0x7a,0x08,0xf2,0x9c,0xac,0x23,0x01,0x00,0x00 diff --git a/llvm/test/MC/ELF/reloc-directive.s b/llvm/test/MC/ELF/reloc-directive.s index 59d7ace40d7e06d7ea56ac9c898e8b9ec1791282..a4658f938d0d38a66823187e190fa9c28dd10a1e 100644 --- a/llvm/test/MC/ELF/reloc-directive.s +++ b/llvm/test/MC/ELF/reloc-directive.s @@ -10,11 +10,14 @@ # ASM-NEXT: .Ltmp2: # ASM-NEXT: .reloc 2+.Ltmp2, R_X86_64_NONE, foo # ASM-NEXT: .reloc (1+foo)+3, R_X86_64_NONE, data+1 +# ASM-NEXT: .Ltmp3: +# ASM-NEXT: .reloc .Ltmp3, BFD_RELOC_NONE, unused # CHECK: 0x2 R_X86_64_NONE foo 0x0 # CHECK-NEXT: 0x0 R_X86_64_NONE foo 0x0 # CHECK-NEXT: 0x3 R_X86_64_NONE foo 0x0 # CHECK-NEXT: 0x4 R_X86_64_NONE data 0x1 +# CHECK-NEXT: 0x1 R_X86_64_NONE unused 0x0 .text .globl foo @@ -24,6 +27,7 @@ foo: .reloc .-1, R_X86_64_NONE, foo .reloc 2+., R_X86_64_NONE, foo .reloc 1+foo+3, R_X86_64_NONE, data+1 + .reloc ., BFD_RELOC_NONE, unused .data .globl data diff --git a/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s index d2d0d44f639a968081fbafc646679f4ae5aacc40..03179480147a511c3fcd4cc3ff556e68204b3039 100644 --- a/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s +++ b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s @@ -1044,3 +1044,301 @@ # CHECK-BE: hashchkp 5, -512(1) # encoding: [0x7c,0x01,0x2d,0x64] # CHECK-LE: hashchkp 5, -512(1) # encoding: [0x64,0x2d,0x01,0x7c] hashchkp 5, -512(1) + +## +## Extended Mnemonics +## + +# CHECK-BE: wait # encoding: [0x7c,0x00,0x00,0x3c] +# CHECK-LE: wait # encoding: [0x3c,0x00,0x00,0x7c] + wait +# CHECK-BE: wait # encoding: [0x7c,0x00,0x00,0x3c] +# CHECK-LE: wait # encoding: [0x3c,0x00,0x00,0x7c] + wait 0 +# CHECK-BE: waitrsv # encoding: [0x7c,0x20,0x00,0x3c] +# CHECK-LE: waitrsv # encoding: [0x3c,0x00,0x20,0x7c] + wait 1 +# CHECK-BE: waitrsv # encoding: [0x7c,0x20,0x00,0x3c] +# CHECK-LE: waitrsv # encoding: [0x3c,0x00,0x20,0x7c] + waitrsv +# CHECK-BE: wait 2 0 # encoding: [0x7c,0x40,0x00,0x3c] +# CHECK-LE: wait 2 0 # encoding: [0x3c,0x00,0x40,0x7c] + pause_short + +# CHECK-BE: sync # encoding: [0x7c,0x00,0x04,0xac] +# CHECK-LE: sync # encoding: [0xac,0x04,0x00,0x7c] + sync +# CHECK-BE: ptesync # encoding: [0x7c,0x40,0x04,0xac] +# CHECK-LE: ptesync # encoding: [0xac,0x04,0x40,0x7c] + sync 2 +# CHECK-BE: sync # encoding: [0x7c,0x00,0x04,0xac] +# CHECK-LE: sync # encoding: [0xac,0x04,0x00,0x7c] + hwsync +# CHECK-BE: sync 1 # encoding: [0x7c,0x20,0x04,0xac] +# CHECK-LE: sync 1 # encoding: [0xac,0x04,0x20,0x7c] + wsync +# CHECK-BE: ptesync # encoding: [0x7c,0x40,0x04,0xac] +# CHECK-LE: ptesync # encoding: [0xac,0x04,0x40,0x7c] + ptesync +# CHECK-BE: phwsync # encoding: [0x7c,0x80,0x04,0xac] +# CHECK-LE: phwsync # encoding: [0xac,0x04,0x80,0x7c] + phwsync +# CHECK-BE: plwsync # encoding: [0x7c,0xa0,0x04,0xac] +# CHECK-LE: plwsync # encoding: [0xac,0x04,0xa0,0x7c] + plwsync +# CHECK-BE: stncisync # encoding: [0x7c,0x21,0x04,0xac] +# CHECK-LE: stncisync # encoding: [0xac,0x04,0x21,0x7c] + stncisync +# CHECK-BE: stcisync # encoding: [0x7c,0x02,0x04,0xac] +# CHECK-LE: stcisync # encoding: [0xac,0x04,0x02,0x7c] + stcisync +# CHECK-BE: stsync # encoding: [0x7c,0x03,0x04,0xac] +# CHECK-LE: stsync # encoding: [0xac,0x04,0x03,0x7c] + stsync + +# CHECK-BE: paddi 4, 5, 4294965249 # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0x38,0x85,0xf8,0x01] +# CHECK-LE: paddi 4, 5, 4294965249 # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0x85,0x38] + paddi 4, 5, 4294965249 +# CHECK-BE: pli 4, 4294965249 # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0x38,0x80,0xf8,0x01] +# CHECK-LE: pli 4, 4294965249 # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0x80,0x38] + pli 4, 4294965249 +# CHECK-BE: paddi 4, 3, 4294965249 # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0x38,0x83,0xf8,0x01] +# CHECK-LE: paddi 4, 3, 4294965249 # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0x83,0x38] + pla 4, 4294965249(3) +# CHECK-BE: paddi 4, 0, 4294965249, 1 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0x38,0x80,0xf8,0x01] +# CHECK-LE: paddi 4, 0, 4294965249, 1 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0x80,0x38] + pla 4, 4294965249 +# CHECK-BE: paddi 4, 5, -80, 0 # encoding: [0x06,0x03,0xff,0xff, +# CHECK-BE-SAME: 0x38,0x85,0xff,0xb0] +# CHECK-LE: paddi 4, 5, -80, 0 # encoding: [0xff,0xff,0x03,0x06, +# CHECK-LE-SAME: 0xb0,0xff,0x85,0x38] + psubi 4, 5, 80 + +# CHECK-BE: plbz 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0x88,0xa3,0xf8,0x01] +# CHECK-LE: plbz 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0x88] + plbz 5, 4294965249(3) +# CHECK-BE: plbz 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0x88,0xa0,0xf8,0x01] +# CHECK-LE: plbz 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0x88] + plbz 5, 4294965249 +# CHECK-BE: plhz 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xa0,0xa3,0xf8,0x01] +# CHECK-LE: plhz 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xa0] + plhz 5, 4294965249(3) +# CHECK-BE: plhz 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xa0,0xa0,0xf8,0x01] +# CHECK-LE: plhz 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xa0] + plhz 5, 4294965249 +# CHECK-BE: plha 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xa8,0xa3,0xf8,0x01] +# CHECK-LE: plha 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xa8] + plha 5, 4294965249(3) +# CHECK-BE: plha 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xa8,0xa0,0xf8,0x01] +# CHECK-LE: plha 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xa8] + plha 5, 4294965249 +# CHECK-BE: plwz 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0x80,0xa3,0xf8,0x01] +# CHECK-LE: plwz 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0x80] + plwz 5, 4294965249(3) +# CHECK-BE: plwz 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0x80,0xa0,0xf8,0x01] +# CHECK-LE: plwz 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0x80] + plwz 5, 4294965249 +# CHECK-BE: plwa 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xa4,0xa3,0xf8,0x01] +# CHECK-LE: plwa 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xa4] + plwa 5, 4294965249(3) +# CHECK-BE: plwa 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xa4,0xa0,0xf8,0x01] +# CHECK-LE: plwa 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xa4] + plwa 5, 4294965249 +# CHECK-BE: pld 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xe4,0xa3,0xf8,0x01] +# CHECK-LE: pld 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xe4] + pld 5, 4294965249(3) +# CHECK-BE: pld 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xe4,0xa0,0xf8,0x01] +# CHECK-LE: pld 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xe4] + pld 5, 4294965249 +# CHECK-BE: pstb 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0x98,0xa3,0xf8,0x01] +# CHECK-LE: pstb 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0x98] + pstb 5, 4294965249(3) +# CHECK-BE: pstb 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0x98,0xa0,0xf8,0x01] +# CHECK-LE: pstb 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0x98] + pstb 5, 4294965249 +# CHECK-BE: psth 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xb0,0xa3,0xf8,0x01] +# CHECK-LE: psth 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xb0] + psth 5, 4294965249(3) +# CHECK-BE: psth 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xb0,0xa0,0xf8,0x01] +# CHECK-LE: psth 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xb0] + psth 5, 4294965249 +# CHECK-BE: pstw 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0x90,0xa3,0xf8,0x01] +# CHECK-LE: pstw 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0x90] + pstw 5, 4294965249(3) +# CHECK-BE: pstw 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0x90,0xa0,0xf8,0x01] +# CHECK-LE: pstw 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0x90] + pstw 5, 4294965249 +# CHECK-BE: pstd 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xf4,0xa3,0xf8,0x01] +# CHECK-LE: pstd 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xf4] + pstd 5, 4294965249(3) +# CHECK-BE: pstd 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xf4,0xa0,0xf8,0x01] +# CHECK-LE: pstd 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xf4] + pstd 5, 4294965249 +# CHECK-BE: plfs 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xc0,0xa3,0xf8,0x01] +# CHECK-LE: plfs 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xc0] + plfs 5, 4294965249(3) +# CHECK-BE: plfs 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xc0,0xa0,0xf8,0x01] +# CHECK-LE: plfs 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xc0] + plfs 5, 4294965249 +# CHECK-BE: plfd 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xc8,0xa3,0xf8,0x01] +# CHECK-LE: plfd 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xc8] + plfd 5, 4294965249(3) +# CHECK-BE: plfd 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xc8,0xa0,0xf8,0x01] +# CHECK-LE: plfd 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xc8] + plfd 5, 4294965249 +# CHECK-BE: pstfs 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xd0,0xa3,0xf8,0x01] +# CHECK-LE: pstfs 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xd0] + pstfs 5, 4294965249(3) +# CHECK-BE: pstfs 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xd0,0xa0,0xf8,0x01] +# CHECK-LE: pstfs 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xd0] + pstfs 5, 4294965249 +# CHECK-BE: pstfd 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xd8,0xa3,0xf8,0x01] +# CHECK-LE: pstfd 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xd8] + pstfd 5, 4294965249(3) +# CHECK-BE: pstfd 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xd8,0xa0,0xf8,0x01] +# CHECK-LE: pstfd 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xd8] + pstfd 5, 4294965249 +# CHECK-BE: plxsd 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xa8,0xa3,0xf8,0x01] +# CHECK-LE: plxsd 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xa8] + plxsd 5, 4294965249(3) +# CHECK-BE: plxsd 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xa8,0xa0,0xf8,0x01] +# CHECK-LE: plxsd 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xa8] + plxsd 5, 4294965249 +# CHECK-BE: plxssp 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xac,0xa3,0xf8,0x01] +# CHECK-LE: plxssp 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xac] + plxssp 5, 4294965249(3) +# CHECK-BE: plxssp 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xac,0xa0,0xf8,0x01] +# CHECK-LE: plxssp 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xac] + plxssp 5, 4294965249 +# CHECK-BE: plxv 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xc8,0xa3,0xf8,0x01] +# CHECK-LE: plxv 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xc8] + plxv 5, 4294965249(3) +# CHECK-BE: plxv 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xc8,0xa0,0xf8,0x01] +# CHECK-LE: plxv 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xc8] + plxv 5, 4294965249 +# CHECK-BE: plxvp 6, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xe8,0xc3,0xf8,0x01] +# CHECK-LE: plxvp 6, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xc3,0xe8] + plxvp 6, 4294965249(3) +# CHECK-BE: plxvp 6, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xe8,0xc0,0xf8,0x01] +# CHECK-LE: plxvp 6, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xc0,0xe8] + plxvp 6, 4294965249 +# CHECK-BE: pstxsd 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xb8,0xa3,0xf8,0x01] +# CHECK-LE: pstxsd 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xb8] + pstxsd 5, 4294965249(3) +# CHECK-BE: pstxsd 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xb8,0xa0,0xf8,0x01] +# CHECK-LE: pstxsd 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xb8] + pstxsd 5, 4294965249 +# CHECK-BE: pstxssp 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xbc,0xa3,0xf8,0x01] +# CHECK-LE: pstxssp 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xbc] + pstxssp 5, 4294965249(3) +# CHECK-BE: pstxssp 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xbc,0xa0,0xf8,0x01] +# CHECK-LE: pstxssp 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xbc] + pstxssp 5, 4294965249 +# CHECK-BE: pstxv 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xd8,0xa3,0xf8,0x01] +# CHECK-LE: pstxv 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xd8] + pstxv 5, 4294965249(3) +# CHECK-BE: pstxv 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xd8,0xa0,0xf8,0x01] +# CHECK-LE: pstxv 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xd8] + pstxv 5, 4294965249 +# CHECK-BE: pstxvp 6, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xf8,0xc3,0xf8,0x01] +# CHECK-LE: pstxvp 6, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xc3,0xf8] + pstxvp 6, 4294965249(3) +# CHECK-BE: pstxvp 6, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xf8,0xc0,0xf8,0x01] +# CHECK-LE: pstxvp 6, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xc0,0xf8] + pstxvp 6, 4294965249 diff --git a/llvm/test/MC/PowerPC/ppc64-errors.s b/llvm/test/MC/PowerPC/ppc64-errors.s index 627ae410db88210e0398eb58ec5ebf41bcd20134..1c03dc084acdff9957fb5de5dd245ea8b166e0cd 100644 --- a/llvm/test/MC/PowerPC/ppc64-errors.s +++ b/llvm/test/MC/PowerPC/ppc64-errors.s @@ -53,8 +53,8 @@ wait 4 # CHECK: error: invalid operand for instruction -# CHECK-NEXT: sync 4 - sync 4 +# CHECK-NEXT: sync 8 + sync 8 # Unsigned 3-bit immediate operands diff --git a/llvm/test/MC/X86/apx/invpcid-att.s b/llvm/test/MC/X86/apx/invpcid-att.s new file mode 100644 index 0000000000000000000000000000000000000000..a0e827ecadb0573d508a4168cf14b18c915e9e4c --- /dev/null +++ b/llvm/test/MC/X86/apx/invpcid-att.s @@ -0,0 +1,8 @@ +# RUN: llvm-mc -triple x86_64 --show-encoding %s | FileCheck %s +# RUN: not llvm-mc -triple i386 -show-encoding %s 2>&1 | FileCheck %s --check-prefix=ERROR + +# ERROR-COUNT-1: error: +# ERROR-NOT: error: +# CHECK: invpcid 291(%r28,%r29,4), %r19 +# CHECK: encoding: [0x62,0x8c,0x7a,0x08,0xf2,0x9c,0xac,0x23,0x01,0x00,0x00] + invpcid 291(%r28,%r29,4), %r19 diff --git a/llvm/test/MC/X86/apx/invpcid-intel.s b/llvm/test/MC/X86/apx/invpcid-intel.s new file mode 100644 index 0000000000000000000000000000000000000000..e9993d26962e22de68ff0f8835e6b1f5b38fd3da --- /dev/null +++ b/llvm/test/MC/X86/apx/invpcid-intel.s @@ -0,0 +1,5 @@ +# RUN: llvm-mc -triple x86_64 -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s + +# CHECK: invpcid r19, xmmword ptr [r28 + 4*r29 + 291] +# CHECK: encoding: [0x62,0x8c,0x7a,0x08,0xf2,0x9c,0xac,0x23,0x01,0x00,0x00] + invpcid r19, xmmword ptr [r28 + 4*r29 + 291] diff --git a/llvm/test/MachineVerifier/test_g_prefetch.mir b/llvm/test/MachineVerifier/test_g_prefetch.mir new file mode 100644 index 0000000000000000000000000000000000000000..a08b0803fc3559abf9992ee72d265e1c73902a35 --- /dev/null +++ b/llvm/test/MachineVerifier/test_g_prefetch.mir @@ -0,0 +1,40 @@ +# RUN: not --crash llc -o - -mtriple=aarch64 -global-isel -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s +# REQUIRES: aarch64-registered-target + +--- +name: test_fcmp +legalized: true +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: +body: | + bb.0: + liveins: $x0, $w0, $q0 + %s32:_(s32) = COPY $w0 + %ptr:_(p0) = COPY $x0 + + G_PREFETCH %ptr + ; CHECK: *** Bad machine code: Too few operands *** + ; CHECK: 4 operands expected, but 1 given. + + G_PREFETCH %ptr, 0, 0, 0, 0 + ; CHECK: *** Bad machine code: Extra explicit operand on non-variadic instruction *** + ; CHECK: operand 4: + + G_PREFETCH %s32, 0, 0, 0 + ; CHECK: *** Bad machine code: addr operand must be a pointer *** + ; CHECK: operand 0: + + G_PREFETCH %ptr, 10, 0, 0 + ; CHECK: *** Bad machine code: rw operand must be an immediate 0-1 *** + ; CHECK: operand 1: + + G_PREFETCH %ptr, 0, 10, 0 + ; CHECK: *** Bad machine code: locality operand must be an immediate 0-3 *** + ; CHECK: operand 2: + + G_PREFETCH %ptr, 0, 0, 10 + ; CHECK: *** Bad machine code: cache type operand must be an immediate 0-1 *** + ; CHECK: operand 3: +... diff --git a/llvm/test/TableGen/ContextlessPredicates.td b/llvm/test/TableGen/ContextlessPredicates.td index 7f081e9a0ec00a277488720d4583c71f3bcb24ca..0f4c4d0c450e1b5864ace9ae49f50166dee69e8b 100644 --- a/llvm/test/TableGen/ContextlessPredicates.td +++ b/llvm/test/TableGen/ContextlessPredicates.td @@ -26,7 +26,7 @@ def : Pat<(test_atomic_op_frag GPR32:$ptr, GPR32:$val) , // CHECK_NOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK_NOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ATOMICRMW_XCHG, // CHECK_NOPT-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/4, -// CHECK_NOPT-NEXT: // MIs[0] dst +// CHECK_NOPT-NEXT: // MIs[0] DstI[dst] // CHECK_NOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK_NOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // CHECK_NOPT-NEXT: // MIs[0] ptr diff --git a/llvm/test/TableGen/DefaultOpsGlobalISel.td b/llvm/test/TableGen/DefaultOpsGlobalISel.td index 13ee2631ecb078414af3161bbb89751d6d37a74e..c997467c570ad885d4ed430a5bd8f3ee6d17b5be 100644 --- a/llvm/test/TableGen/DefaultOpsGlobalISel.td +++ b/llvm/test/TableGen/DefaultOpsGlobalISel.td @@ -35,7 +35,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckComplexPattern, /*MI*/0, /*Op*/1, /*Renderer*/0, GICP_gi_SelectSrcMods, // CHECK: GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/1, GICP_gi_SelectSrcMods, // CHECK: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FMAX, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // mods0 // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0 // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/1, /*SubOperand*/1, // mods1 @@ -48,7 +48,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckComplexPattern, /*MI*/0, /*Op*/1, /*Renderer*/0, GICP_gi_SelectClampOMod, // CHECK: // (ffloor:{ *:[f32] } (SelectClampOMod:{ *:[f32] } f32:{ *:[f32] }:$src0, omod:{ *:[i32] }:$omod, i1:{ *:[i1] }:$clamp)) => (FLOMP:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp, omod:{ *:[i32] }:$omod) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FLOMP, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0 // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/2, // clamp // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // omod @@ -56,7 +56,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FCANONICALIZE, // CHECK: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FMAX, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // mods // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // mods @@ -68,7 +68,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FCOS, // CHECK: // (fcos:{ *:[f32] } (SelectOMod:{ *:[f32] } f32:{ *:[f32] }:$src0, i32:{ *:[i32] }:$omod)) => (FLAMP:{ *:[f32] } FPR32:{ *:[f32] }:$src0, omod:{ *:[i32] }:$omod) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FLAMP, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0 // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // omod // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0, @@ -84,7 +84,7 @@ def clamp : OperandWithDefaultOps ; // CHECK-NEXT: GIR_AddImm, /*InsnID*/1, /*Imm*/0, // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FEEPLE, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0 // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // clamp @@ -95,7 +95,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FSIN, // CHECK: // (fsin:{ *:[f32] } (SelectClamp:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp)) => (FFOO:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FFOO, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0 // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // clamp // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -104,7 +104,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FSQRT, // CHECK: // (fsqrt:{ *:[f32] } (SelectClamp:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp)) => (FLAMP:{ *:[f32] } FPR32:{ *:[f32] }:$src0, 93:{ *:[i32] }, clamp:{ *:[i1] }:$clamp) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FLAMP, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0 // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/93, // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // clamp @@ -113,7 +113,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC_ROUND, // CHECK: // (fround:{ *:[f32] } f32:{ *:[f32] }:$src0) => (FBAR:{ *:[f32] } f32:{ *:[f32] }:$src0) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FBAR, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0 // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -121,7 +121,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC_TRUNC, // CHECK: // (ftrunc:{ *:[f32] } f32:{ *:[f32] }:$src0) => (FFOO:{ *:[f32] } FPR32:{ *:[f32] }:$src0) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FFOO, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0 // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td index 2d968977701fdab6825950b40c73058f9099b023..6ae1305aa1aa51699770b1a7a75c4b87cae4e7c6 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td @@ -28,11 +28,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const int64_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static int64_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/65, 180, /*)*//*default:*//*Label 2*/ 192, -// CHECK-NEXT: /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// CHECK-NEXT: /*TargetOpcode::G_FNEG*//*Label 1*/ 165, -// CHECK-NEXT: // Label 0: @120 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ 164, // Rule ID 1 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/65, 181, /*)*//*default:*//*Label 2*/ 193, +// CHECK-NEXT: /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// CHECK-NEXT: /*TargetOpcode::G_FNEG*//*Label 1*/ 166, +// CHECK-NEXT: // Label 0: @121 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ 165, // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled, // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: // MIs[0] a @@ -57,10 +57,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_ReplaceRegWithTempReg, /*OldInsnID*/0, /*OldOpIdx*/1, /*TempRegID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 3: @164 +// CHECK-NEXT: // Label 3: @165 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @165 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ 191, // Rule ID 0 // +// CHECK-NEXT: // Label 1: @166 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ 192, // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled, // CHECK-NEXT: // MIs[0] dst // CHECK-NEXT: // No operand predicates @@ -75,9 +75,9 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_ReplaceReg, /*OldInsnID*/0, /*OldOpIdx*/0, /*NewInsnId*/1, /*NewOpIdx*/1, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 4: @191 +// CHECK-NEXT: // Label 4: @192 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @192 +// CHECK-NEXT: // Label 2: @193 // CHECK-NEXT: GIM_Reject, // CHECK-NEXT: }; // CHECK-NEXT: return MatchTable0; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td index 0495a66a7c577d1266f1f5222c898c2f3144c2dc..fd5f7db0b4f12d8298871e0dae4070ccf61cc18b 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td @@ -34,12 +34,12 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const int64_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static int64_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/19, 126, /*)*//*default:*//*Label 3*/ 194, -// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// CHECK-NEXT: /*TargetOpcode::G_CONSTANT*//*Label 1*/ 138, 0, 0, 0, 0, 0, -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 2*/ 165, -// CHECK-NEXT: // Label 0: @112 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ 137, // Rule ID 0 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/19, 127, /*)*//*default:*//*Label 3*/ 195, +// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// CHECK-NEXT: /*TargetOpcode::G_CONSTANT*//*Label 1*/ 139, 0, 0, 0, 0, 0, +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 2*/ 166, +// CHECK-NEXT: // Label 0: @113 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ 138, // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled, // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] a @@ -51,10 +51,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 4: @137 +// CHECK-NEXT: // Label 4: @138 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @138 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ 164, // Rule ID 2 // +// CHECK-NEXT: // Label 1: @139 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ 165, // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule2Enabled, // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] a @@ -66,10 +66,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddCImm, /*InsnID*/0, /*Type*/GILLT_s32, /*Imm*/42, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 5: @164 +// CHECK-NEXT: // Label 5: @165 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @165 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ 193, // Rule ID 1 // +// CHECK-NEXT: // Label 2: @166 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ 194, // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled, // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates @@ -83,9 +83,9 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 6: @193 +// CHECK-NEXT: // Label 6: @194 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @194 +// CHECK-NEXT: // Label 3: @195 // CHECK-NEXT: GIM_Reject, // CHECK-NEXT: }; // CHECK-NEXT: return MatchTable0; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td index 5cb9206ca5f2caf1213f5fcf5e99ac93ec42144b..b62ebcf24687437b122692c7c9248592d737aa56 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td @@ -28,7 +28,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const int64_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static int64_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/118, 181, /*)*//*default:*//*Label 3*/ 152, +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/119, 182, /*)*//*default:*//*Label 3*/ 152, // CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 0*/ 68, 0, 0, 0, 0, 0, 0, // CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 1*/ 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // CHECK-NEXT: /*TargetOpcode::G_FPEXT*//*Label 2*/ 127, diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td index a74f7fbbe1cce697a5becdb7fed100087b4316a5..6777089f846f2dcde7fdb739e018e6356831b339 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td @@ -132,15 +132,15 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // Verify match table. // CHECK: const int64_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static int64_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/19, 126, /*)*//*default:*//*Label 6*/ 267, -// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ 216, 0, 0, 0, 0, -// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ 231, 0, -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ 239, -// CHECK-NEXT: // Label 0: @112 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ 133, // Rule ID 4 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/19, 127, /*)*//*default:*//*Label 6*/ 268, +// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ 217, 0, 0, 0, 0, +// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ 232, 0, +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ 240, +// CHECK-NEXT: // Label 0: @113 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ 134, // Rule ID 4 // // CHECK-NEXT: GIM_CheckFeatures, GIFBS_HasAnswerToEverything, // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule3Enabled, // CHECK-NEXT: // MIs[0] a @@ -155,8 +155,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // Combiner Rule #3: InstTest1 // CHECK-NEXT: GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 7: @133 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ 140, // Rule ID 3 // +// CHECK-NEXT: // Label 7: @134 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ 141, // Rule ID 3 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule2Enabled, // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates @@ -165,10 +165,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // Combiner Rule #2: InstTest0 // CHECK-NEXT: GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner1, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 8: @140 +// CHECK-NEXT: // Label 8: @141 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @141 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ 180, // Rule ID 6 // +// CHECK-NEXT: // Label 1: @142 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ 181, // Rule ID 6 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule5Enabled, // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] dst @@ -186,10 +186,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // z // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 9: @180 +// CHECK-NEXT: // Label 9: @181 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @181 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ 215, // Rule ID 5 // +// CHECK-NEXT: // Label 2: @182 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ 216, // Rule ID 5 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule4Enabled, // CHECK-NEXT: // MIs[0] tmp // CHECK-NEXT: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/0, // MIs[1] @@ -207,32 +207,32 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner2, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 10: @215 +// CHECK-NEXT: // Label 10: @216 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @216 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ 223, // Rule ID 0 // +// CHECK-NEXT: // Label 3: @217 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ 224, // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled, // CHECK-NEXT: // Combiner Rule #0: WipOpcodeTest0; wip_match_opcode 'G_TRUNC' // CHECK-NEXT: GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 11: @223 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ 230, // Rule ID 1 // +// CHECK-NEXT: // Label 11: @224 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ 231, // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled, // CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_TRUNC' // CHECK-NEXT: GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 12: @230 +// CHECK-NEXT: // Label 12: @231 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 4: @231 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ 238, // Rule ID 2 // +// CHECK-NEXT: // Label 4: @232 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ 239, // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled, // CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_SEXT' // CHECK-NEXT: GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 13: @238 +// CHECK-NEXT: // Label 13: @239 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 5: @239 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ 266, // Rule ID 7 // +// CHECK-NEXT: // Label 5: @240 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ 267, // Rule ID 7 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule6Enabled, // CHECK-NEXT: // MIs[0] dst // CHECK-NEXT: // No operand predicates @@ -247,10 +247,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 14: @266 +// CHECK-NEXT: // Label 14: @267 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 6: @267 +// CHECK-NEXT: // Label 6: @268 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; +// CHECK-NEXT: }; // Size: 2152 bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelEmitter-input-discard.td b/llvm/test/TableGen/GlobalISelEmitter-input-discard.td index 6d0d1de5d06c037c505354625dff120803a2f802..68a9553a8b7381602918f4092ed931e936da503b 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-input-discard.td +++ b/llvm/test/TableGen/GlobalISelEmitter-input-discard.td @@ -21,7 +21,7 @@ def FOO : I<(outs GPR32:$dst), (ins GPR32Op:$src0, GPR32Op:$src1), []>; // GISEL-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define, // GISEL-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FOO, -// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // GISEL-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/3, // src1 // GISEL-NEXT: GIR_EraseFromParent, /*InsnID*/0, diff --git a/llvm/test/TableGen/GlobalISelEmitter-multiple-output-discard.td b/llvm/test/TableGen/GlobalISelEmitter-multiple-output-discard.td index 5f442b067d9cde9b070881eaf5e8e8c2225dc9c9..64cf31f3772ee93d34756001a1695273e2648060 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-multiple-output-discard.td +++ b/llvm/test/TableGen/GlobalISelEmitter-multiple-output-discard.td @@ -22,10 +22,10 @@ def : GINodeEquiv; def : Pat<(two_out GPR32:$val), (THREE_OUTS GPR32:$val)>; // CHECK: GIM_CheckOpcode, /*MI*/0, MyTarget::G_TWO_OUT, -// CHECK-NEXT: // MIs[0] out1 +// CHECK-NEXT: // MIs[0] DstI[out1] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, -// CHECK-NEXT: // MIs[0] out2 +// CHECK-NEXT: // MIs[0] DstI[out2] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: // MIs[0] val @@ -34,8 +34,8 @@ def : Pat<(two_out GPR32:$val), (THREE_OUTS GPR32:$val)>; // CHECK-NEXT: // (two_out:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$val) => (THREE_OUTS:{ *:[i32] }:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$val) // CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::THREE_OUTS, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // out1 -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // out2 +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[out1] +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // DstI[out2] // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/RegState::Define|RegState::Dead, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // val // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, diff --git a/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td b/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td index c98ac73c61af7be6105dcfc8793a6d8b78a5b5c9..85ac3ace0364ccbbe3d9627122d145b81779aae4 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td +++ b/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td @@ -30,10 +30,10 @@ def : Pat<(loadpost (p0 GPR32:$addr), (i32 GPR32:$off)), >; // CHECK: GIM_CheckOpcode, /*MI*/0, MyTarget::G_POST_LOAD, -// CHECK-NEXT: // MIs[0] val +// CHECK-NEXT: // MIs[0] DstI[val] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, -// CHECK-NEXT: // MIs[0] ptr_out +// CHECK-NEXT: // MIs[0] DstI[ptr_out] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_p0s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: // MIs[0] addr @@ -64,10 +64,10 @@ def : GINodeEquiv; def : Pat<(two_in GPR32:$i1, GPR32:$i2), (TWO_INS GPR32:$i2, GPR32:$i1)>; // CHECK: GIM_CheckOpcode, /*MI*/0, MyTarget::G_TWO_IN, -// CHECK-NEXT: // MIs[0] out1 +// CHECK-NEXT: // MIs[0] DstI[out1] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, -// CHECK-NEXT: // MIs[0] out2 +// CHECK-NEXT: // MIs[0] DstI[out2] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: // MIs[0] i1 @@ -78,8 +78,8 @@ def : Pat<(two_in GPR32:$i1, GPR32:$i2), (TWO_INS GPR32:$i2, GPR32:$i1)>; // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/3, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: // (two_in:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$i1, GPR32:{ *:[i32] }:$i2) => (TWO_INS:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$i2, GPR32:{ *:[i32] }:$i1) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::TWO_INS, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // out1 -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // out2 +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[out1] +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // DstI[out2] // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/3, // i2 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // i1 // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, diff --git a/llvm/test/TableGen/GlobalISelEmitter-nested-subregs.td b/llvm/test/TableGen/GlobalISelEmitter-nested-subregs.td index c48c82aa142acc9d181d94cd712fad8e4ce0071f..61f79bc04b27e187bf97789471fbfe7bfbcf4243 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-nested-subregs.td +++ b/llvm/test/TableGen/GlobalISelEmitter-nested-subregs.td @@ -32,7 +32,7 @@ def A0 : RegisterClass<"MyTarget", [i32], 32, (add a0)>; // CHECK: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ANYEXT, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s16, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::A0RegClassID, // CHECK-NEXT: // MIs[0] src @@ -52,7 +52,7 @@ def A0 : RegisterClass<"MyTarget", [i32], 32, (add a0)>; // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, MyTarget::A0RegClassID, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/2, MyTarget::A0bRegClassID, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempSubRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, MyTarget::lo16, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, MyTarget::A0wRegClassID, diff --git a/llvm/test/TableGen/GlobalISelEmitter-output-discard.td b/llvm/test/TableGen/GlobalISelEmitter-output-discard.td index c755d8377e61da3cdf77c1b26d4c8ff23a687cc9..1cfe49dee0970466913fb446afb35a428e68f512 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-output-discard.td +++ b/llvm/test/TableGen/GlobalISelEmitter-output-discard.td @@ -15,7 +15,7 @@ def ADD_CO : I<(outs GPR32:$dst, GPR8:$flag), // GISEL-NEXT: // (add:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1) => (ADD_CO:{ *:[i32] }:{ *:[i8] } GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1) // GISEL-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s8, // GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::ADD_CO, -// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // GISEL-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/RegState::Define|RegState::Dead, // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src1 diff --git a/llvm/test/TableGen/GlobalISelEmitter-zero-reg.td b/llvm/test/TableGen/GlobalISelEmitter-zero-reg.td index 374430bc427e82c446bddd9b1e4fd3bb33859cf0..63ad5d5cd03d965f789c4a76e03017465cad0494 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-zero-reg.td +++ b/llvm/test/TableGen/GlobalISelEmitter-zero-reg.td @@ -24,7 +24,7 @@ def INST : PredI<(outs GPR32:$dst), (ins GPR32:$src), []>; // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, // CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: // MIs[0] src @@ -32,7 +32,7 @@ def INST : PredI<(outs GPR32:$dst), (ins GPR32:$src), []>; // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: // (ld:{ *:[i32] } GPR32:{ *:[i32] }:$src)<><> => (INST:{ *:[i32] } GPR32:{ *:[i32] }:$src) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INST, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src // CHECK-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::NoRegister, /*AddRegisterRegFlags*/0, // CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*MergeInsnID's*/0, GIU_MergeMemOperands_EndOfList, diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td index b7a81894f6442fa789d02c3a3449f2e70b00924f..562fdefcd3d6c0e4a420d3c9c8ba24f428b67a68 100644 --- a/llvm/test/TableGen/GlobalISelEmitter.td +++ b/llvm/test/TableGen/GlobalISelEmitter.td @@ -247,7 +247,7 @@ def HasC : Predicate<"Subtarget->hasC()"> { let RecomputePerFunction = 1; } // R19O-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID, // R19N-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/4, // R19N-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SELECT, -// R19N-NEXT: // MIs[0] dst +// R19N-NEXT: // MIs[0] DstI[dst] // R19N-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // R19N-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // R19N-NEXT: // MIs[0] src1 @@ -290,7 +290,7 @@ def HasC : Predicate<"Subtarget->hasC()"> { let RecomputePerFunction = 1; } // R19C-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/1, /*RendererID*/2, /*SubOperand*/1, // src5b // R19C-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // R19C-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSN3, -// R19C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// R19C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // R19C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 // R19C-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // src2b // R19C-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src2a @@ -341,7 +341,7 @@ def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b), // R21O-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID, // R21N-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/4, // R21N-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SELECT, -// R21N-NEXT: // MIs[0] dst +// R21N-NEXT: // MIs[0] DstI[dst] // R21N-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // R21N-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // R21N-NEXT: // MIs[0] src1 @@ -359,7 +359,7 @@ def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b), // R21C-NEXT: // (select:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2, complex:{ *:[i32] }:$src3)<> => (INSN2:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src3, complex:{ *:[i32] }:$src2) // R21C-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSN2, -// R21C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// R21C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // R21C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 // R21C-NEXT: GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/1, // R21C-NEXT: GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/0, @@ -397,7 +397,7 @@ def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b), // // R20N-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // R20N-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SUB, -// R20N-NEXT: // MIs[0] dst +// R20N-NEXT: // MIs[0] DstI[dst] // R20N-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // R20N-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // R20N-NEXT: // MIs[0] src1 @@ -410,7 +410,7 @@ def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b), // R20C-NEXT: GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/0, GICP_gi_complex, // R20C-NEXT: // (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2) => (INSN1:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2) // R20C-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSN1, -// R20C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// R20C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // R20C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 // R20C-NEXT: GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/0, // R20C-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -462,7 +462,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3), // R00C-NEXT: GIM_CheckFeatures, GIFBS_HasA, // R00N-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // R00N-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SUB, -// R00N-NEXT: // MIs[0] dst +// R00N-NEXT: // MIs[0] DstI[dst] // R00N-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // R00N-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // R00N-NEXT: // MIs[0] Operand 1 @@ -501,7 +501,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3), // R00C-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/2, // R00C-NEXT: // (sub:{ *:[i32] } (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2), (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src3, GPR32:{ *:[i32] }:$src4)) => (INSNBOB:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src3, GPR32:{ *:[i32] }:$src4) // R00C-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSNBOB, -// R00C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// R00C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // R00C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src1 // R00C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // src2 // R00C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/2, /*OpIdx*/1, // src3 @@ -517,7 +517,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3), // R00O-NEXT: GIM_Reject, // R00O: // Label [[DEFAULT_NUM]]: @[[DEFAULT]] // R00O-NEXT: GIM_Reject, -// R00O-NEXT: }; +// R00O-NEXT: }; // Size: 9888 bytes def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4), [(set GPR32:$dst, @@ -544,7 +544,7 @@ def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, G // R01O-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // // R01N-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC, -// R01N-NEXT: // MIs[0] dst +// R01N-NEXT: // MIs[0] DstI[dst] // R01N-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // R01N-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // R01N-NEXT: // MIs[0] Operand 1 @@ -555,7 +555,7 @@ def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, G // R01C-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID, // R01C-NEXT: // (intrinsic_wo_chain:{ *:[i32] } [[ID:[0-9]+]]:{ *:[iPTR] }, GPR32:{ *:[i32] }:$src1) => (MOV:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // R01C-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOV, -// R01C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// R01C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // R01C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src1 // R01C-NEXT: GIR_EraseFromParent, /*InsnID*/0, // R01C-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -591,7 +591,7 @@ def MOV : I<(outs GPR32:$dst), (ins GPR32:$src1), // // R02N-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // R02N-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR, -// R02N-NEXT: // MIs[0] dst +// R02N-NEXT: // MIs[0] DstI[dst] // R02N-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // R02N-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // R02N-NEXT: // MIs[0] src1 @@ -603,7 +603,7 @@ def MOV : I<(outs GPR32:$dst), (ins GPR32:$src1), // R02C-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -2 // R02C-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -2:{ *:[i32] }) => (XORI:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // R02C-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XORI, -// R02C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// R02C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // R02C-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/-1, // R02C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 // R02C-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -624,7 +624,7 @@ def XORI : I<(outs GPR32:$dst), (ins m1:$src2, GPR32:$src1), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -635,7 +635,7 @@ def XORI : I<(outs GPR32:$dst), (ins m1:$src2, GPR32:$src1), // NOOPT-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -3 // NOOPT-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -3:{ *:[i32] }) => (XOR:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XOR, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::R0, // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -653,7 +653,7 @@ def XOR : I<(outs GPR32:$dst), (ins Z:$src2, GPR32:$src1), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -664,7 +664,7 @@ def XOR : I<(outs GPR32:$dst), (ins Z:$src2, GPR32:$src1), // NOOPT-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -4 // NOOPT-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -4:{ *:[i32] }) => (XORlike:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XORlike, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/-1, // NOOPT-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::R0, // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 @@ -683,7 +683,7 @@ def XORlike : I<(outs GPR32:$dst), (ins m1Z:$src2, GPR32:$src1), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -694,7 +694,7 @@ def XORlike : I<(outs GPR32:$dst), (ins m1Z:$src2, GPR32:$src1), // NOOPT-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -5, // NOOPT-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -5:{ *:[i32] }) => (XORManyDefaults:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XORManyDefaults, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/-1, // NOOPT-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::R0, // NOOPT-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::R0, @@ -717,7 +717,7 @@ def XORManyDefaults : I<(outs GPR32:$dst), (ins m1Z:$src3, Z:$src2, GPR32:$src1) // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] Wm @@ -728,7 +728,7 @@ def XORManyDefaults : I<(outs GPR32:$dst), (ins m1Z:$src3, Z:$src2, GPR32:$src1) // NOOPT-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -1, // NOOPT-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$Wm, -1:{ *:[i32] }) => (ORN:{ *:[i32] } R0:{ *:[i32] }, GPR32:{ *:[i32] }:$Wm) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::ORN, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::R0, // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // Wm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -746,7 +746,7 @@ def : Pat<(not GPR32:$Wm), (ORN R0, GPR32:$Wm)>; // NOOPT-NEXT: GIM_CheckFeatures, GIFBS_HasA, // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] Operand 1 @@ -768,7 +768,7 @@ def : Pat<(not GPR32:$Wm), (ORN R0, GPR32:$Wm)>; // NOOPT-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, // NOOPT-NEXT: // (mul:{ *:[i32] } (add:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2), GPR32:{ *:[i32] }:$src3) => (MULADD:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src3) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MULADD, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src1 // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // src2 // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src3 @@ -784,7 +784,7 @@ def : Pat<(not GPR32:$Wm), (ORN R0, GPR32:$Wm)>; // NOOPT-NEXT: GIM_CheckFeatures, GIFBS_HasA, // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src3 @@ -806,13 +806,13 @@ def : Pat<(not GPR32:$Wm), (ORN R0, GPR32:$Wm)>; // NOOPT-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, // NOOPT-NEXT: // (mul:{ *:[i32] } GPR32:{ *:[i32] }:$src3, (add:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2)) => (MULADD:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src3) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MULADD, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src1 // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // src2 // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src3 // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 26, +// NOOPT-NEXT: // GIR_Coverage, 27, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -826,14 +826,14 @@ def MULADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] Operand 1 // NOOPT-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, 1, // NOOPT-NEXT: // 1:{ *:[i32] } => (MOV1:{ *:[i32] }) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOV1, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, // NOOPT-NEXT: // GIR_Coverage, 7, @@ -848,14 +848,14 @@ def MOV1 : I<(outs GPR32:$dst), (ins), [(set GPR32:$dst, 1)]>; // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT, // NOOPT-NEXT: GIM_CheckI64ImmPredicate, /*MI*/0, /*Predicate*/GICXXPred_I64_Predicate_simm8, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] Operand 1 // NOOPT-NEXT: // No operand predicates // NOOPT-NEXT: // (imm:{ *:[i32] })<>:$imm => (MOVimm8:{ *:[i32] } (imm:{ *:[i32] }):$imm) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVimm8, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_CopyConstantAsSImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -872,14 +872,14 @@ def MOVimm8 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, simm8:$i // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT, // NOOPT-NEXT: GIM_CheckAPIntImmPredicate, /*MI*/0, /*Predicate*/GICXXPred_APInt_Predicate_simm9, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] Operand 1 // NOOPT-NEXT: // No operand predicates // NOOPT-NEXT: // (imm:{ *:[i32] })<>:$imm => (MOVimm9:{ *:[i32] } (imm:{ *:[i32] }):$imm) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVimm9, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_CopyConstantAsSImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -896,14 +896,14 @@ def MOVimm9 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, simm9:$i // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT, // NOOPT-NEXT: GIM_CheckI64ImmPredicate, /*MI*/0, /*Predicate*/GICXXPred_I64_Predicate_cimm8, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] Operand 1 // NOOPT-NEXT: // No operand predicates // NOOPT-NEXT: // (imm:{ *:[i32] })<><>:$imm => (MOVcimm8:{ *:[i32] } (cimm8_xform:{ *:[i32] } (imm:{ *:[i32] }):$imm)) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVcimm8, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_CustomRenderer, /*InsnID*/0, /*OldInsnID*/0, /*Renderer*/GICR_renderImm, // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -919,14 +919,14 @@ def MOVcimm8 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, cimm8:$ // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FCONSTANT, // NOOPT-NEXT: GIM_CheckAPFloatImmPredicate, /*MI*/0, /*Predicate*/GICXXPred_APFloat_Predicate_fpimmz, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::FPR32RegClassID, // NOOPT-NEXT: // MIs[0] Operand 1 // NOOPT-NEXT: // No operand predicates // NOOPT-NEXT: // (fpimm:{ *:[f32] })<>:$imm => (MOVfpimmz:{ *:[f32] } (fpimm:{ *:[f32] }):$imm) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVfpimmz, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_CopyFConstantAsFPImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -941,7 +941,7 @@ def MOVcimm8 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, cimm8:$ // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, // NOOPT-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // NOOPT-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -964,7 +964,7 @@ def LOAD : I<(outs GPR32:$dst), (ins GPR32:$src1), // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, // NOOPT-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // NOOPT-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_p0s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src @@ -987,7 +987,7 @@ def : Pat<(load GPR32:$src), // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SEXTLOAD, // NOOPT-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/2, // NOOPT-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -1008,7 +1008,7 @@ def SEXTLOAD : I<(outs GPR32:$dst), (ins GPR32:$src1), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -1032,7 +1032,7 @@ def ADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src{{$}} @@ -1042,7 +1042,7 @@ def ADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2), // NOOPT-NEXT: GIM_CheckIsSameOperand, /*MI*/0, /*OpIdx*/2, /*OtherMI*/0, /*OtherOpIdx*/1, // NOOPT-NEXT: // (add:{ *:[i32] } GPR32:{ *:[i32] }:$src, GPR32:{ *:[i32] }:$src) => (DOUBLE:{ *:[i32] } GPR32:{ *:[i32] }:$src) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::DOUBLE, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -1052,12 +1052,41 @@ def ADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2), def DOUBLE : I<(outs GPR32:$dst), (ins GPR32:$src), [(set GPR32:$dst, (add GPR32:$src, GPR32:$src))]>; +//===- Test a pattern with unintended operand name clash. ----------------===// + +// Check that using the same name for +// - Def operand of the instruction corresponding to the root node of the +// pattern's destination +// - one of operands in the pattern itself +// does not introduce unexpected GIM_CheckIsSameOperand predicate. + +// NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], +// NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, +// NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD, +// NOOPT-NEXT: // MIs[0] DstI[samename] +// NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, +// NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, +// NOOPT-NEXT: // MIs[0] samename +// NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, +// NOOPT-NEXT: // MIs[0] othername +// NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, +// NOOPT-NEXT: // (add:{ *:[i32] } i32:{ *:[i32] }:$samename, i32:{ *:[i32] }:$othername) => (InsnWithSpeciallyNamedDef:{ *:[i32] } i32:{ *:[i32] }:$samename, i32:{ *:[i32] }:$othername) +// NOOPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::InsnWithSpeciallyNamedDef, +// NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, +// NOOPT-NEXT: // GIR_Coverage, 24, +// NOOPT-NEXT: GIR_Done, +// NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] + +def InsnWithSpeciallyNamedDef : I<(outs GPR32:$samename), (ins GPR32:$src1, GPR32:$src2), []>; +def : Pat<(add i32:$samename, i32:$othername), + (InsnWithSpeciallyNamedDef i32:$samename, i32:$othername)>; + //===- Test a simple pattern with ValueType operands. ----------------------===// // // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -1067,7 +1096,7 @@ def DOUBLE : I<(outs GPR32:$dst), (ins GPR32:$src), [(set GPR32:$dst, (add GPR32 // NOOPT-NEXT: // (add:{ *:[i32] } i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2) => (ADD:{ *:[i32] } i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2) // NOOPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::ADD, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 24, +// NOOPT-NEXT: // GIR_Coverage, 25, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -1080,7 +1109,7 @@ def : Pat<(add i32:$src1, i32:$src2), // NOOPT-NEXT: GIM_CheckFeatures, GIFBS_HasA_HasB_HasC, // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -1091,7 +1120,7 @@ def : Pat<(add i32:$src1, i32:$src2), // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // (mul:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2) => (MUL:{ *:[i32] } GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src1) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MUL, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src2 // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -1110,7 +1139,7 @@ def MUL : I<(outs GPR32:$dst), (ins GPR32:$src2, GPR32:$src1), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_BITCAST, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -1119,7 +1148,7 @@ def MUL : I<(outs GPR32:$dst), (ins GPR32:$src2, GPR32:$src1), // NOOPT-NEXT: // (bitconvert:{ *:[i32] } FPR32:{ *:[f32] }:$src1) => (COPY_TO_REGCLASS:{ *:[i32] } FPR32:{ *:[f32] }:$src1, GPR32:{ *:[i32] }) // NOOPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/TargetOpcode::COPY, // NOOPT-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, MyTarget::GPR32RegClassID, -// NOOPT-NEXT: // GIR_Coverage, 25, +// NOOPT-NEXT: // GIR_Coverage, 26, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -1131,14 +1160,14 @@ def : Pat<(i32 (bitconvert FPR32:$src1)), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] Operand 1 // NOOPT-NEXT: // No operand predicates // NOOPT-NEXT: // (imm:{ *:[i32] }):$imm => (MOVimm:{ *:[i32] } (imm:{ *:[i32] }):$imm) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVimm, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_CopyConstantAsSImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -1169,5 +1198,5 @@ def BR : I<(outs), (ins unknown:$target), [(br bb:$target)]>; // NOOPT-NEXT: GIM_Reject, -// NOOPT-NEXT: }; +// NOOPT-NEXT: }; // Size: 11408 bytes // NOOPT-NEXT: return MatchTable0; diff --git a/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td b/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td index d07ef4e300ee5fc3f24703b19e5a4f6080def3d7..5a75cb129eec1253ee57c7eab3dc8b06816d87d6 100644 --- a/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td +++ b/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td @@ -76,7 +76,7 @@ def and_or_pat : PatFrag< // CHECK: GIM_Try, /*On fail goto*//*Label 0*/ 99, // Rule ID 7 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] src2 @@ -106,7 +106,7 @@ def and_or_pat : PatFrag< // CHECK: GIM_Try, /*On fail goto*//*Label 1*/ 198, // Rule ID 3 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] Operand 1 @@ -151,7 +151,7 @@ def mul_pat : PatFrag< // CHECK: GIM_Try, /*On fail goto*//*Label 2*/ 293, // Rule ID 4 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] Operand 1 @@ -180,7 +180,7 @@ def mul_pat : PatFrag< // CHECK: GIM_Try, /*On fail goto*//*Label 3*/ 388, // Rule ID 8 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] src2 @@ -226,7 +226,7 @@ def sub3_pat : PatFrag< // CHECK: GIM_Try, /*On fail goto*//*Label 4*/ 475, // Rule ID 0 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SUB, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] Operand 1 diff --git a/llvm/test/TableGen/GlobalISelEmitterHwModes.td b/llvm/test/TableGen/GlobalISelEmitterHwModes.td index b185feaf009fb920f49e4cc5a65b13e800bfa676..04f33648313b2f7aacb67efd1b593c2a1f5545b0 100644 --- a/llvm/test/TableGen/GlobalISelEmitterHwModes.td +++ b/llvm/test/TableGen/GlobalISelEmitterHwModes.td @@ -133,7 +133,7 @@ class I Pat> // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, // CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s64, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPRRegClassID, // CHECK-NEXT: // MIs[0] src1 @@ -151,7 +151,7 @@ class I Pat> // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, // CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPRRegClassID, // CHECK-NEXT: // MIs[0] src1 @@ -175,7 +175,7 @@ def LOAD : I<(outs GPR:$dst), (ins GPR:$src1), // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, // CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_p0s64, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPRRegClassID, // CHECK-NEXT: // MIs[0] src @@ -193,7 +193,7 @@ def LOAD : I<(outs GPR:$dst), (ins GPR:$src1), // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, // CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_p0s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPRRegClassID, // CHECK-NEXT: // MIs[0] src diff --git a/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizer.td b/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizer.td index cd7a177b8426a26567a5cba41f6681420eabba54..6a369b6a7b8897f4fa771aae4033df97a32f039d 100644 --- a/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizer.td +++ b/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizer.td @@ -61,7 +61,7 @@ def LOAD16Imm : I<(outs GPR16:$dst), (ins GPR16:$src), []>; // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, // CHECK-NEXT: // (ld:{ *:[i16] } (add:{ *:[i16] } GPR16:{ *:[i16] }:$src, 10:{ *:[i16] }))<><> => (LOAD16Imm:{ *:[i16] } GPR16:{ *:[i16] }:$src) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::LOAD16Imm, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src // CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*MergeInsnID's*/0, 1, GIU_MergeMemOperands_EndOfList, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, diff --git a/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand-invalid.td b/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand-invalid.td index 44e1b08bdd8299de4d28fd87acf0d7e4c08aafa3..2423c3bd32d5068977f97c80d1c17cd799150792 100644 --- a/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand-invalid.td +++ b/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand-invalid.td @@ -33,7 +33,7 @@ def InstThreeOperands : I<(outs GPR32:$dst), (ins GPR32:$cond, GPR32:$src,GPR32: // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/2, // CHECK-NEXT: // (select:{ *:[i32] } (setcc:{ *:[i32] } GPR32:{ *:[i32] }:$cond, 0:{ *:[i32] }, SETEQ:{ *:[Other] }), (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2), GPR32:{ *:[i32] }:$src2) => (InstThreeOperands:{ *:[i32] } GPR32:{ *:[i32] }:$cond, GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::InstThreeOperands, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // cond // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/2, /*OpIdx*/1, // src1 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/2, /*OpIdx*/2, // src2 @@ -61,7 +61,7 @@ def InstThreeOperands : I<(outs GPR32:$dst), (ins GPR32:$cond, GPR32:$src,GPR32: // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/2, // CHECK-NEXT: // (select:{ *:[i32] } (setcc:{ *:[i32] } GPR32:{ *:[i32] }:$cond, 0:{ *:[i32] }, SETNE:{ *:[Other] }), (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2), GPR32:{ *:[i32] }:$src2) => (InstThreeOperands:{ *:[i32] } GPR32:{ *:[i32] }:$cond, GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::InstThreeOperands, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // cond // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/2, /*OpIdx*/1, // src1 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/2, /*OpIdx*/2, // src2 diff --git a/llvm/test/TableGen/GlobalISelEmitterRegSequence.td b/llvm/test/TableGen/GlobalISelEmitterRegSequence.td index 1b7391497f125dbaa1688d67800e66682633f919..34783e688399dec67ef0796276a55605d6ebafb6 100644 --- a/llvm/test/TableGen/GlobalISelEmitterRegSequence.td +++ b/llvm/test/TableGen/GlobalISelEmitterRegSequence.td @@ -32,7 +32,7 @@ def SUBSOME_INSN : I<(outs SRegs:$dst), (ins SOP:$src), []>; // CHECK: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SEXT, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] src @@ -50,7 +50,7 @@ def SUBSOME_INSN : I<(outs SRegs:$dst), (ins SOP:$src), []>; // CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/1, // src // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::REG_SEQUENCE, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*SubRegIndex*/1, // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/1, /*TempRegFlags*/0, diff --git a/llvm/test/TableGen/GlobalISelEmitterSubreg.td b/llvm/test/TableGen/GlobalISelEmitterSubreg.td index c8405735aaea0c59a1775a63ac9961b14a620d24..af3db1590368b7a26882c3bf4e35dc5ff0490b8e 100644 --- a/llvm/test/TableGen/GlobalISelEmitterSubreg.td +++ b/llvm/test/TableGen/GlobalISelEmitterSubreg.td @@ -72,7 +72,7 @@ def : Pat<(sub (complex DOP:$src1, DOP:$src2), 77), // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, Test::SRegsRegClassID, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, Test::DRegsRegClassID, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SOME_INSN2, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/1, /*TempRegFlags*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -87,7 +87,7 @@ def : Pat<(i32 (anyext i16:$src)), (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SOP:$src // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define, // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::INSERT_SUBREG, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/1, @@ -115,7 +115,7 @@ def : Pat<(i32 (anyext i16:$src)), (SOME_INSN (INSERT_SUBREG (i32 (IMPLICIT_DEF) // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, Test::DRegsRegClassID, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/2, Test::SRegsRegClassID, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SOME_INSN, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -145,7 +145,7 @@ def : Pat<(i32 (anyext i16:$src)), (INSERT_SUBREG (i32 (IMPLICIT_DEF)), (SUBSOME // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define, // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::INSERT_SUBREG, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/1, /*TempRegFlags*/0, // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/1, @@ -182,7 +182,7 @@ def : Pat<(i16 (trunc (bitreverse DOP:$src))), // CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/1, /*OpIdx*/1, // src // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempSubRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, sub0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, Test::SRegsRegClassID, @@ -207,7 +207,7 @@ def : Pat<(i16 (trunc (bitreverse DOP:$src))), // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, Test::SRegsRegClassID, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, Test::DRegsRegClassID, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SUBSOME_INSN2, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -219,7 +219,7 @@ def : Pat<(i16 (trunc DOP:$src)), (EXTRACT_SUBREG DOP:$src, sub0)>; // CHECK-LABEL: // (trunc:{ *:[i16] } DOP:{ *:[i32] }:$src) => (EXTRACT_SUBREG:{ *:[i16] } DOP:{ *:[i32] }:$src, sub0:{ *:[i32] }) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_CopySubReg, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, /*SubRegIdx*/1, // src // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, Test::SRegsRegClassID, @@ -236,7 +236,7 @@ def : Pat<(i32 (zext SOP:$src)), // CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/1, // src // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::SUBREG_TO_REG, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0, // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/1, diff --git a/llvm/test/TableGen/gisel-physreg-input.td b/llvm/test/TableGen/gisel-physreg-input.td index e700141c203efa4759e71bd7f96e72398b076896..3dbd57bc45b7c0a9bffa19471d2b3748487fcdcb 100644 --- a/llvm/test/TableGen/gisel-physreg-input.td +++ b/llvm/test/TableGen/gisel-physreg-input.td @@ -27,7 +27,7 @@ class I Pat> // GISEL: GIM_Try, // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD, -// GISEL-NEXT: // MIs[0] dst +// GISEL-NEXT: // MIs[0] DstI[dst] // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // GISEL-NEXT: // MIs[0] src0 @@ -41,7 +41,7 @@ class I Pat> // GISEL-NEXT: GIR_AddRegister, /*InsnID*/1, MyTarget::SPECIAL, /*AddRegisterRegFlags*/RegState::Define, // GISEL-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/2, // SPECIAL // GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::ADD_PHYS, -// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0 // GISEL-NEXT: GIR_EraseFromParent, /*InsnID*/0, // GISEL-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -55,7 +55,7 @@ def ADD_PHYS : I<(outs GPR32:$dst), (ins GPR32:$src0), // GISEL: GIM_Try, // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL, -// GISEL-NEXT: // MIs[0] dst +// GISEL-NEXT: // MIs[0] DstI[dst] // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // GISEL-NEXT: // MIs[0] SPECIAL @@ -69,7 +69,7 @@ def ADD_PHYS : I<(outs GPR32:$dst), (ins GPR32:$src0), // GISEL-NEXT: GIR_AddRegister, /*InsnID*/1, MyTarget::SPECIAL, /*AddRegisterRegFlags*/RegState::Define, // GISEL-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/2, // SPECIAL // GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MUL_PHYS, -// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // SPECIAL // GISEL-NEXT: GIR_EraseFromParent, /*InsnID*/0, // GISEL-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, diff --git a/llvm/test/Transforms/AlignmentFromAssumptions/alignment-from-assumptions-track-users.ll b/llvm/test/Transforms/AlignmentFromAssumptions/alignment-from-assumptions-track-users.ll new file mode 100644 index 0000000000000000000000000000000000000000..1c388528ae4b1902aa04b52775d8e2d31412cab7 --- /dev/null +++ b/llvm/test/Transforms/AlignmentFromAssumptions/alignment-from-assumptions-track-users.ll @@ -0,0 +1,204 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes=alignment-from-assumptions -S | FileCheck %s + +define void @widget(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(3) nocapture %arg1) { +; CHECK-LABEL: define void @widget( +; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(3) nocapture [[ARG1:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG]], i64 1 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[GETELEMENTPTR]], i64 4) ] +; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(1) [[GETELEMENTPTR]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR2:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[ARG1]], i32 1 +; CHECK-NEXT: store i32 [[LOAD]], ptr addrspace(3) [[GETELEMENTPTR2]], align 4 +; CHECK-NEXT: ret void +; +bb: + %getelementptr = getelementptr inbounds i32, ptr addrspace(1) %arg, i64 1 + call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %getelementptr, i64 4) ] + %load = load i32, ptr addrspace(1) %getelementptr, align 2 + %getelementptr2 = getelementptr inbounds i32, ptr addrspace(3) %arg1, i32 1 + store i32 %load, ptr addrspace(3) %getelementptr2, align 4 + ret void +} + +define void @wibble(ptr addrspace(1) nocapture readonly %arg, i32 %arg2, ptr addrspace(3) nocapture %arg3) { +; CHECK-LABEL: define void @wibble( +; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) nocapture [[ARG3:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[ICMP:%.*]] = icmp ugt i32 [[ARG2]], 10 +; CHECK-NEXT: br i1 [[ICMP]], label [[BB4:%.*]], label [[BB5:%.*]] +; CHECK: bb4: +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 6 +; CHECK-NEXT: br label [[BB7:%.*]] +; CHECK: bb5: +; CHECK-NEXT: [[GETELEMENTPTR6:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 7 +; CHECK-NEXT: br label [[BB7]] +; CHECK: bb7: +; CHECK-NEXT: [[PHI:%.*]] = phi ptr addrspace(1) [ [[GETELEMENTPTR]], [[BB4]] ], [ [[GETELEMENTPTR6]], [[BB5]] ] +; CHECK-NEXT: [[GETELEMENTPTR8:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[PHI]], i64 4 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[ARG]], i64 4) ] +; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(1) [[GETELEMENTPTR8]], align 2 +; CHECK-NEXT: [[GETELEMENTPTR9:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[ARG3]], i32 1 +; CHECK-NEXT: store i32 [[LOAD]], ptr addrspace(3) [[GETELEMENTPTR9]], align 4 +; CHECK-NEXT: ret void +; +bb: + %icmp = icmp ugt i32 %arg2, 10 + br i1 %icmp, label %bb4, label %bb5 + +bb4: ; preds = %bb + %getelementptr = getelementptr i32, ptr addrspace(1) %arg, i32 6 + br label %bb7 + +bb5: ; preds = %bb + %getelementptr6 = getelementptr i32, ptr addrspace(1) %arg, i32 7 + br label %bb7 + +bb7: ; preds = %bb5, %bb4 + %phi = phi ptr addrspace(1) [ %getelementptr, %bb4 ], [ %getelementptr6, %bb5 ] + %getelementptr8 = getelementptr inbounds i32, ptr addrspace(1) %phi, i64 4 + call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %arg, i64 4) ] + %load = load i32, ptr addrspace(1) %getelementptr8, align 2 + %getelementptr9 = getelementptr inbounds i32, ptr addrspace(3) %arg3, i32 1 + store i32 %load, ptr addrspace(3) %getelementptr9, align 4 + ret void +} + +define void @ham(ptr addrspace(1) nocapture readonly %arg, i32 %arg2, ptr addrspace(3) nocapture %arg3) { +; CHECK-LABEL: define void @ham( +; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) nocapture [[ARG3:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 0 +; CHECK-NEXT: [[GETELEMENTPTR4:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 10 +; CHECK-NEXT: [[ICMP:%.*]] = icmp ugt i32 [[ARG2]], 10 +; CHECK-NEXT: br i1 [[ICMP]], label [[BB5:%.*]], label [[BB10:%.*]] +; CHECK: bb5: +; CHECK-NEXT: [[PHI:%.*]] = phi ptr addrspace(1) [ [[GETELEMENTPTR]], [[BB:%.*]] ], [ [[GETELEMENTPTR8:%.*]], [[BB5]] ] +; CHECK-NEXT: [[PHI6:%.*]] = phi i32 [ 0, [[BB]] ], [ [[ADD:%.*]], [[BB5]] ] +; CHECK-NEXT: [[GETELEMENTPTR7:%.*]] = getelementptr i32, ptr addrspace(1) [[PHI]], i32 4 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[ARG]], i64 4) ] +; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(1) [[GETELEMENTPTR7]], align 4 +; CHECK-NEXT: [[ADD]] = add i32 [[PHI6]], [[LOAD]] +; CHECK-NEXT: [[GETELEMENTPTR8]] = getelementptr i32, ptr addrspace(1) [[PHI]], i32 [[ARG2]] +; CHECK-NEXT: [[ICMP9:%.*]] = icmp eq ptr addrspace(1) [[GETELEMENTPTR8]], [[GETELEMENTPTR4]] +; CHECK-NEXT: br i1 [[ICMP9]], label [[BB5]], label [[BB10]] +; CHECK: bb10: +; CHECK-NEXT: [[PHI11:%.*]] = phi i32 [ 0, [[BB]] ], [ [[ADD]], [[BB5]] ] +; CHECK-NEXT: [[GETELEMENTPTR12:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[ARG3]], i32 1 +; CHECK-NEXT: store i32 [[PHI11]], ptr addrspace(3) [[GETELEMENTPTR12]], align 4 +; CHECK-NEXT: ret void +; +bb: + %getelementptr = getelementptr i32, ptr addrspace(1) %arg, i32 0 + %getelementptr4 = getelementptr i32, ptr addrspace(1) %arg, i32 10 + %icmp = icmp ugt i32 %arg2, 10 + br i1 %icmp, label %bb5, label %bb10 + +bb5: ; preds = %bb5, %bb + %phi = phi ptr addrspace(1) [ %getelementptr, %bb ], [ %getelementptr8, %bb5 ] + %phi6 = phi i32 [ 0, %bb ], [ %add, %bb5 ] + %getelementptr7 = getelementptr i32, ptr addrspace(1) %phi, i32 4 + call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %arg, i64 4) ] + %load = load i32, ptr addrspace(1) %getelementptr7, align 2 + %add = add i32 %phi6, %load + %getelementptr8 = getelementptr i32, ptr addrspace(1) %phi, i32 %arg2 + %icmp9 = icmp eq ptr addrspace(1) %getelementptr8, %getelementptr4 + br i1 %icmp9, label %bb5, label %bb10 + +bb10: ; preds = %bb5, %bb + %phi11 = phi i32 [ 0, %bb ], [ %add, %bb5 ] + %getelementptr12 = getelementptr inbounds i32, ptr addrspace(3) %arg3, i32 1 + store i32 %phi11, ptr addrspace(3) %getelementptr12, align 4 + ret void +} + +define void @quux(ptr addrspace(1) nocapture readonly %arg, i32 %arg2, ptr addrspace(3) nocapture %arg3) { +; CHECK-LABEL: define void @quux( +; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) nocapture [[ARG3:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[ICMP:%.*]] = icmp ugt i32 [[ARG2]], 10 +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 6 +; CHECK-NEXT: [[GETELEMENTPTR4:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 7 +; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[ICMP]], ptr addrspace(1) [[GETELEMENTPTR]], ptr addrspace(1) [[GETELEMENTPTR4]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[ARG]], i64 4) ] +; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(1) [[SELECT]], align 2 +; CHECK-NEXT: [[GETELEMENTPTR5:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[ARG3]], i32 1 +; CHECK-NEXT: store i32 [[LOAD]], ptr addrspace(3) [[GETELEMENTPTR5]], align 4 +; CHECK-NEXT: ret void +; +bb: + %icmp = icmp ugt i32 %arg2, 10 + %getelementptr = getelementptr i32, ptr addrspace(1) %arg, i32 6 + %getelementptr4 = getelementptr i32, ptr addrspace(1) %arg, i32 7 + %select = select i1 %icmp, ptr addrspace(1) %getelementptr, ptr addrspace(1) %getelementptr4 + call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %arg, i64 4) ] + %load = load i32, ptr addrspace(1) %select, align 2 + %getelementptr5 = getelementptr inbounds i32, ptr addrspace(3) %arg3, i32 1 + store i32 %load, ptr addrspace(3) %getelementptr5, align 4 + ret void +} + +define void @widget.1(ptr addrspace(1) nocapture readonly %arg, i32 %arg2, ptr addrspace(3) nocapture %arg3) { +; CHECK-LABEL: define void @widget.1( +; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) nocapture [[ARG3:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[ADDRSPACECAST:%.*]] = addrspacecast ptr addrspace(3) [[ARG3]] to ptr addrspace(1) +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr addrspace(1) [[ADDRSPACECAST]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr addrspace(3) [[ARG3]], i64 4) ] +; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(1) [[GETELEMENTPTR]], align 2 +; CHECK-NEXT: [[GETELEMENTPTR4:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[ARG3]], i32 1 +; CHECK-NEXT: store i32 [[LOAD]], ptr addrspace(3) [[GETELEMENTPTR4]], align 4 +; CHECK-NEXT: ret void +; +bb: + %addrspacecast = addrspacecast ptr addrspace(3) %arg3 to ptr addrspace(1) + %getelementptr = getelementptr i32, ptr addrspace(1) %addrspacecast + call void @llvm.assume(i1 true) [ "align"(ptr addrspace(3) %arg3, i64 4) ] + %load = load i32, ptr addrspace(1) %getelementptr, align 2 + %getelementptr4 = getelementptr inbounds i32, ptr addrspace(3) %arg3, i32 1 + store i32 %load, ptr addrspace(3) %getelementptr4, align 2 + ret void +} + +define void @baz(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(3) nocapture %arg1) { +; CHECK-LABEL: define void @baz( +; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(3) nocapture [[ARG1:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr ptr addrspace(1), ptr addrspace(1) [[ARG]], i64 16 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[ARG]], i64 4) ] +; CHECK-NEXT: [[LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(1) [[GETELEMENTPTR]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR2:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[ARG1]], i32 1 +; CHECK-NEXT: store ptr addrspace(1) [[LOAD]], ptr addrspace(3) [[GETELEMENTPTR2]], align 2 +; CHECK-NEXT: ret void +; +bb: + %getelementptr = getelementptr ptr addrspace(1), ptr addrspace(1) %arg, i64 16 + call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %arg, i64 4) ] + %load = load ptr addrspace(1), ptr addrspace(1) %getelementptr, align 2 + %getelementptr2 = getelementptr inbounds i32, ptr addrspace(3) %arg1, i32 1 + store ptr addrspace(1) %load, ptr addrspace(3) %getelementptr2, align 2 + ret void +} + +define void @foo(ptr addrspace(1) nocapture readonly %arg, i32 %arg1) { +; CHECK-LABEL: define void @foo( +; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG1:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr ptr addrspace(3), ptr addrspace(1) [[ARG]], i64 16 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[ARG]], i64 4) ] +; CHECK-NEXT: [[LOAD:%.*]] = load ptr addrspace(3), ptr addrspace(1) [[GETELEMENTPTR]], align 4 +; CHECK-NEXT: store i32 [[ARG1]], ptr addrspace(3) [[LOAD]], align 2 +; CHECK-NEXT: ret void +; +bb: + %getelementptr = getelementptr ptr addrspace(3), ptr addrspace(1) %arg, i64 16 + call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %arg, i64 4) ] + %load = load ptr addrspace(3), ptr addrspace(1) %getelementptr, align 2 + store i32 %arg1, ptr addrspace(3) %load, align 2 + ret void +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) +declare void @llvm.assume(i1 noundef) #0 + +attributes #0 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) } diff --git a/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll b/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll index c28fc59014f5c6f2c250c9804f5782e22109ea0f..58dff360ff6a5c00f8333a94676dee5dbf83d22a 100644 --- a/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll +++ b/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll @@ -9,7 +9,7 @@ define i32 @test_01(i32 %A, i64 %Len, ptr %array) { ; CHECK-NEXT: br i1 [[TRIPCHECK]], label [[LOOP_PREHEADER:%.*]], label [[ZERO:%.*]] ; CHECK: loop.preheader: ; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[A:%.*]] to i64 -; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1 ; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[LEN]], i64 0) ; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[LEN]], [[SMIN]] ; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP2]], i64 [[TMP1]]) diff --git a/llvm/test/Transforms/IndVarSimplify/promote-iv-to-eliminate-casts.ll b/llvm/test/Transforms/IndVarSimplify/promote-iv-to-eliminate-casts.ll index 60e014b0efca53aa656e5dd89815594275e343aa..3f0ada281b1e340a3523c98bb7e44ccadeef33ce 100644 --- a/llvm/test/Transforms/IndVarSimplify/promote-iv-to-eliminate-casts.ll +++ b/llvm/test/Transforms/IndVarSimplify/promote-iv-to-eliminate-casts.ll @@ -182,7 +182,7 @@ define void @promote_latch_condition_decrementing_loop_01(ptr %p, ptr %a) { ; CHECK-LABEL: @promote_latch_condition_decrementing_loop_01( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[LEN:%.*]] = load i32, ptr [[P:%.*]], align 4, !range [[RNG0:![0-9]+]] -; CHECK-NEXT: [[LEN_MINUS_1:%.*]] = add i32 [[LEN]], -1 +; CHECK-NEXT: [[LEN_MINUS_1:%.*]] = add nsw i32 [[LEN]], -1 ; CHECK-NEXT: [[ZERO_CHECK:%.*]] = icmp eq i32 [[LEN]], 0 ; CHECK-NEXT: br i1 [[ZERO_CHECK]], label [[LOOPEXIT:%.*]], label [[PREHEADER:%.*]] ; CHECK: preheader: diff --git a/llvm/test/Transforms/InstCombine/alloca.ll b/llvm/test/Transforms/InstCombine/alloca.ll index a64de28ee8397738f7610585c6a03ee156bb8ecd..4247d6a3019312060c54946a2ebbce6cbbd983ad 100644 --- a/llvm/test/Transforms/InstCombine/alloca.ll +++ b/llvm/test/Transforms/InstCombine/alloca.ll @@ -248,3 +248,14 @@ entry: call void (...) @use(ptr nonnull @int) [ "blah"(ptr %y) ] ret void } + +define void @test_inalloca_with_element_count(ptr %a) { +; ALL-LABEL: @test_inalloca_with_element_count( +; ALL-NEXT: [[ALLOCA1:%.*]] = alloca inalloca [10 x %struct_type], align 4 +; ALL-NEXT: call void @test9_aux(ptr nonnull inalloca([[STRUCT_TYPE:%.*]]) [[ALLOCA1]]) +; ALL-NEXT: ret void +; + %alloca = alloca inalloca %struct_type, i32 10, align 4 + call void @test9_aux(ptr inalloca(%struct_type) %alloca) + ret void +} diff --git a/llvm/test/Transforms/InstCombine/binop-of-displaced-shifts.ll b/llvm/test/Transforms/InstCombine/binop-of-displaced-shifts.ll index c86dfde6ddece99fcaffd8b99ac27e2a6515f10c..27a3c8743368a156c712c38acac975ceb5f96ce3 100644 --- a/llvm/test/Transforms/InstCombine/binop-of-displaced-shifts.ll +++ b/llvm/test/Transforms/InstCombine/binop-of-displaced-shifts.ll @@ -328,3 +328,32 @@ define <2 x i8> @shl_or_non_splat_out_of_range(<2 x i8> %x) { %binop = or <2 x i8> %shift, %shift2 ret <2 x i8> %binop } + +define i8 @shl_or_with_or_disjoint_instead_of_add(i8 %x) { +; CHECK-LABEL: define i8 @shl_or_with_or_disjoint_instead_of_add +; CHECK-SAME: (i8 [[X:%.*]]) { +; CHECK-NEXT: [[BINOP:%.*]] = shl i8 22, [[X]] +; CHECK-NEXT: ret i8 [[BINOP]] +; + %shift = shl i8 16, %x + %add = or disjoint i8 %x, 1 + %shift2 = shl i8 3, %add + %binop = or i8 %shift, %shift2 + ret i8 %binop +} + +define i8 @shl_or_with_or_instead_of_add(i8 %x) { +; CHECK-LABEL: define i8 @shl_or_with_or_instead_of_add +; CHECK-SAME: (i8 [[X:%.*]]) { +; CHECK-NEXT: [[SHIFT:%.*]] = shl i8 16, [[X]] +; CHECK-NEXT: [[ADD:%.*]] = or i8 [[X]], 1 +; CHECK-NEXT: [[SHIFT2:%.*]] = shl i8 3, [[ADD]] +; CHECK-NEXT: [[BINOP:%.*]] = or i8 [[SHIFT]], [[SHIFT2]] +; CHECK-NEXT: ret i8 [[BINOP]] +; + %shift = shl i8 16, %x + %add = or i8 %x, 1 + %shift2 = shl i8 3, %add + %binop = or i8 %shift, %shift2 + ret i8 %binop +} diff --git a/llvm/test/Transforms/InstCombine/ctlz-cttz-shifts.ll b/llvm/test/Transforms/InstCombine/ctlz-cttz-shifts.ll new file mode 100644 index 0000000000000000000000000000000000000000..86fef51872b1908685c27a24b60fc2ac2676c14a --- /dev/null +++ b/llvm/test/Transforms/InstCombine/ctlz-cttz-shifts.ll @@ -0,0 +1,234 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes=instcombine -S < %s | FileCheck %s + +declare i32 @llvm.ctlz.i32(i32, i1) +declare i32 @llvm.cttz.i32(i32, i1) +declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1) +declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1) + +define i32 @lshr_ctlz_true(i32) { +; CHECK-LABEL: define i32 @lshr_ctlz_true( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTLZ:%.*]] = add i32 [[TMP0]], 9 +; CHECK-NEXT: ret i32 [[CTLZ]] +; + %lshr = lshr i32 8387584, %0 + %ctlz = call i32 @llvm.ctlz.i32(i32 %lshr, i1 true) + ret i32 %ctlz +} + +define i32 @shl_nuw_ctlz_true(i32) { +; CHECK-LABEL: define i32 @shl_nuw_ctlz_true( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTLZ:%.*]] = sub i32 9, [[TMP0]] +; CHECK-NEXT: ret i32 [[CTLZ]] +; + %shl = shl nuw i32 8387584, %0 + %ctlz = call i32 @llvm.ctlz.i32(i32 %shl, i1 true) + ret i32 %ctlz +} + +define i32 @shl_nuw_nsw_ctlz_true(i32) { +; CHECK-LABEL: define i32 @shl_nuw_nsw_ctlz_true( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTLZ:%.*]] = sub i32 9, [[TMP0]] +; CHECK-NEXT: ret i32 [[CTLZ]] +; + %shl = shl nuw nsw i32 8387584, %0 + %ctlz = call i32 @llvm.ctlz.i32(i32 %shl, i1 true) + ret i32 %ctlz +} + +define i32 @lshr_exact_cttz_true(i32) { +; CHECK-LABEL: define i32 @lshr_exact_cttz_true( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTTZ:%.*]] = sub i32 10, [[TMP0]] +; CHECK-NEXT: ret i32 [[CTTZ]] +; + %lshr = lshr exact i32 8387584, %0 + %cttz = call i32 @llvm.cttz.i32(i32 %lshr, i1 true) + ret i32 %cttz +} + +define i32 @shl_cttz_true(i32) { +; CHECK-LABEL: define i32 @shl_cttz_true( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTTZ:%.*]] = add i32 [[TMP0]], 10 +; CHECK-NEXT: ret i32 [[CTTZ]] +; + %shl = shl i32 8387584, %0 + %cttz = call i32 @llvm.cttz.i32(i32 %shl, i1 true) + ret i32 %cttz +} + +define <2 x i32> @vec2_lshr_ctlz_true(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_lshr_ctlz_true( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTLZ:%.*]] = add <2 x i32> [[TMP0]], +; CHECK-NEXT: ret <2 x i32> [[CTLZ]] +; + %div = lshr <2 x i32> , %0 + %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %div, i1 true) + ret <2 x i32> %ctlz +} + +define <2 x i32> @vec2_shl_nuw_ctlz_true(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_shl_nuw_ctlz_true( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTLZ:%.*]] = sub <2 x i32> , [[TMP0]] +; CHECK-NEXT: ret <2 x i32> [[CTLZ]] +; + %shl = shl nuw <2 x i32> , %0 + %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %shl, i1 true) + ret <2 x i32> %ctlz +} + +define <2 x i32> @vec2_shl_nuw_nsw_ctlz_true(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_shl_nuw_nsw_ctlz_true( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTLZ:%.*]] = sub <2 x i32> , [[TMP0]] +; CHECK-NEXT: ret <2 x i32> [[CTLZ]] +; + %shl = shl nuw nsw <2 x i32> , %0 + %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %shl, i1 true) + ret <2 x i32> %ctlz +} + +define <2 x i32> @vec2_lshr_exact_cttz_true(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_lshr_exact_cttz_true( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTTZ:%.*]] = sub <2 x i32> , [[TMP0]] +; CHECK-NEXT: ret <2 x i32> [[CTTZ]] +; + %lshr = lshr exact <2 x i32> , %0 + %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %lshr, i1 true) + ret <2 x i32> %cttz +} + +define <2 x i32> @vec2_shl_cttz_true(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_shl_cttz_true( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTTZ:%.*]] = add <2 x i32> [[TMP0]], +; CHECK-NEXT: ret <2 x i32> [[CTTZ]] +; + %shl = shl <2 x i32> , %0 + %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %shl, i1 true) + ret <2 x i32> %cttz +} + +; negative tests: + +define <2 x i32> @vec2_shl_nsw_ctlz_true_neg(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_shl_nsw_ctlz_true_neg( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[SHL:%.*]] = shl nsw <2 x i32> , [[TMP0]] +; CHECK-NEXT: [[CTLZ:%.*]] = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[SHL]], i1 true), !range [[RNG0:![0-9]+]] +; CHECK-NEXT: ret <2 x i32> [[CTLZ]] +; + %shl = shl nsw <2 x i32> , %0 + %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %shl, i1 true) + ret <2 x i32> %ctlz +} + +define <2 x i32> @vec2_lshr_ctlz_false_neg(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_lshr_ctlz_false_neg( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[DIV:%.*]] = lshr <2 x i32> , [[TMP0]] +; CHECK-NEXT: [[CTLZ:%.*]] = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[DIV]], i1 false), !range [[RNG1:![0-9]+]] +; CHECK-NEXT: ret <2 x i32> [[CTLZ]] +; + %div = lshr <2 x i32> , %0 + %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %div, i1 false) + ret <2 x i32> %ctlz +} + +define <2 x i32> @vec2_shl_ctlz_false_neg(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_shl_ctlz_false_neg( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[SHL:%.*]] = shl <2 x i32> , [[TMP0]] +; CHECK-NEXT: [[CTLZ:%.*]] = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[SHL]], i1 false), !range [[RNG2:![0-9]+]] +; CHECK-NEXT: ret <2 x i32> [[CTLZ]] +; + %shl = shl <2 x i32> , %0 + %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %shl, i1 false) + ret <2 x i32> %ctlz +} + +define <2 x i32> @vec2_lshr_cttz_false_neg(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_lshr_cttz_false_neg( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[LSHR:%.*]] = lshr <2 x i32> , [[TMP0]] +; CHECK-NEXT: [[CTTZ:%.*]] = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[LSHR]], i1 false), !range [[RNG2]] +; CHECK-NEXT: ret <2 x i32> [[CTTZ]] +; + %lshr = lshr <2 x i32> , %0 + %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %lshr, i1 false) + ret <2 x i32> %cttz +} + +define <2 x i32> @vec2_shl_cttz_false_neg(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_shl_cttz_false_neg( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[SHL:%.*]] = shl <2 x i32> , [[TMP0]] +; CHECK-NEXT: [[CTTZ:%.*]] = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[SHL]], i1 false), !range [[RNG3:![0-9]+]] +; CHECK-NEXT: ret <2 x i32> [[CTTZ]] +; + %shl = shl <2 x i32> , %0 + %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %shl, i1 false) + ret <2 x i32> %cttz +} + +define i32 @lshr_ctlz_faslse_neg(i32) { +; CHECK-LABEL: define i32 @lshr_ctlz_faslse_neg( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[LSHR:%.*]] = lshr i32 8387584, [[TMP0]] +; CHECK-NEXT: [[CTLZ:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LSHR]], i1 false), !range [[RNG1]] +; CHECK-NEXT: ret i32 [[CTLZ]] +; + %lshr = lshr i32 8387584, %0 + %ctlz = call i32 @llvm.ctlz.i32(i32 %lshr, i1 false) + ret i32 %ctlz +} + +define i32 @shl_ctlz_false_neg(i32) { +; CHECK-LABEL: define i32 @shl_ctlz_false_neg( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[SHL:%.*]] = shl i32 8387584, [[TMP0]] +; CHECK-NEXT: [[CTLZ:%.*]] = call i32 @llvm.ctlz.i32(i32 [[SHL]], i1 false), !range [[RNG2]] +; CHECK-NEXT: ret i32 [[CTLZ]] +; + %shl = shl i32 8387584, %0 + %ctlz = call i32 @llvm.ctlz.i32(i32 %shl, i1 false) + ret i32 %ctlz +} + +define i32 @lshr_cttz_false_neg(i32) { +; CHECK-LABEL: define i32 @lshr_cttz_false_neg( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[LSHR:%.*]] = lshr i32 8387584, [[TMP0]] +; CHECK-NEXT: [[CTTZ:%.*]] = call i32 @llvm.cttz.i32(i32 [[LSHR]], i1 false), !range [[RNG2]] +; CHECK-NEXT: ret i32 [[CTTZ]] +; + %lshr = lshr i32 8387584, %0 + %cttz = call i32 @llvm.cttz.i32(i32 %lshr, i1 false) + ret i32 %cttz +} + +define i32 @shl_cttz_false_neg(i32) { +; CHECK-LABEL: define i32 @shl_cttz_false_neg( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[SHL:%.*]] = shl i32 8387584, [[TMP0]] +; CHECK-NEXT: [[CTTZ:%.*]] = call i32 @llvm.cttz.i32(i32 [[SHL]], i1 false), !range [[RNG4:![0-9]+]] +; CHECK-NEXT: ret i32 [[CTTZ]] +; + %shl = shl i32 8387584, %0 + %cttz = call i32 @llvm.cttz.i32(i32 %shl, i1 false) + ret i32 %cttz +} +;. +; CHECK: [[RNG0]] = !{i32 1, i32 33} +; CHECK: [[RNG1]] = !{i32 9, i32 33} +; CHECK: [[RNG2]] = !{i32 0, i32 33} +; CHECK: [[RNG3]] = !{i32 3, i32 33} +; CHECK: [[RNG4]] = !{i32 10, i32 33} +;. diff --git a/llvm/test/Transforms/InstCombine/icmp-gep.ll b/llvm/test/Transforms/InstCombine/icmp-gep.ll index 7d266b7b246ae08d28f7cd2442793a7a1c866051..99c784d15eb30ae361f5936cf3c05ff37e5b63e7 100644 --- a/llvm/test/Transforms/InstCombine/icmp-gep.ll +++ b/llvm/test/Transforms/InstCombine/icmp-gep.ll @@ -397,6 +397,52 @@ define i1 @test61_as1(ptr addrspace(1) %foo, i16 %i, i16 %j) { ; Don't transform non-inbounds GEPs. } +define i1 @test60_extra_use(ptr %foo, i64 %i, i64 %j) { +; CHECK-LABEL: @test60_extra_use( +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[FOO:%.*]], i64 [[I:%.*]] +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i16, ptr [[FOO]], i64 [[J:%.*]] +; CHECK-NEXT: call void @use(ptr [[GEP1]]) +; CHECK-NEXT: call void @use(ptr [[GEP2]]) +; CHECK-NEXT: [[CMP:%.*]] = icmp ult ptr [[GEP1]], [[GEP2]] +; CHECK-NEXT: ret i1 [[CMP]] +; + %gep1 = getelementptr inbounds i32, ptr %foo, i64 %i + %gep2 = getelementptr inbounds i16, ptr %foo, i64 %j + call void @use(ptr %gep1) + call void @use(ptr %gep2) + %cmp = icmp ult ptr %gep1, %gep2 + ret i1 %cmp +} + +define i1 @test60_extra_use_const_operands_inbounds(ptr %foo, i64 %i, i64 %j) { +; CHECK-LABEL: @test60_extra_use_const_operands_inbounds( +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[FOO:%.*]], i64 1 +; CHECK-NEXT: call void @use(ptr nonnull [[GEP1]]) +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[J:%.*]], 2 +; CHECK-NEXT: ret i1 [[CMP]] +; + %gep1 = getelementptr inbounds i32, ptr %foo, i64 1 + %gep2 = getelementptr inbounds i16, ptr %foo, i64 %j + call void @use(ptr %gep1) + %cmp = icmp eq ptr %gep1, %gep2 + ret i1 %cmp +} + +define i1 @test60_extra_use_const_operands_no_inbounds(ptr %foo, i64 %i, i64 %j) { +; CHECK-LABEL: @test60_extra_use_const_operands_no_inbounds( +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i32, ptr [[FOO:%.*]], i64 1 +; CHECK-NEXT: call void @use(ptr [[GEP1]]) +; CHECK-NEXT: [[GEP2_IDX_MASK:%.*]] = and i64 [[J:%.*]], 9223372036854775807 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[GEP2_IDX_MASK]], 2 +; CHECK-NEXT: ret i1 [[CMP]] +; + %gep1 = getelementptr i32, ptr %foo, i64 1 + %gep2 = getelementptr i16, ptr %foo, i64 %j + call void @use(ptr %gep1) + %cmp = icmp eq ptr %gep1, %gep2 + ret i1 %cmp +} + define i1 @test_scalable_same(ptr %x) { ; CHECK-LABEL: @test_scalable_same( ; CHECK-NEXT: ret i1 false diff --git a/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll index 574f3e23a4ecf5bcdaaa925ac8227699a55528b5..7f4603881f23c8ffb6d958697c4b87afe1c1cd9a 100644 --- a/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll +++ b/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll @@ -101,7 +101,7 @@ define i1 @both(i8 %bits0, i8 %bits1) { ; CHECK-LABEL: @both( ; CHECK-NEXT: [[T0:%.*]] = shl nsw i8 -1, [[BITS0:%.*]] ; CHECK-NEXT: [[T2:%.*]] = shl nsw i8 -1, [[BITS1:%.*]] -; CHECK-NEXT: [[R:%.*]] = icmp uge i8 [[T2]], [[T0]] +; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[T0]], [[T2]] ; CHECK-NEXT: ret i1 [[R]] ; %t0 = shl i8 -1, %bits0 diff --git a/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll index 5f0af994f0eb82bc840d60a9767a2ef4cf10b7e3..c7a45c5cdc11ad77a242bc74a1f50eb856d0a803 100644 --- a/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll +++ b/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll @@ -101,7 +101,7 @@ define i1 @both(i8 %bits0, i8 %bits1) { ; CHECK-LABEL: @both( ; CHECK-NEXT: [[T0:%.*]] = shl nsw i8 -1, [[BITS0:%.*]] ; CHECK-NEXT: [[T2:%.*]] = shl nsw i8 -1, [[BITS1:%.*]] -; CHECK-NEXT: [[R:%.*]] = icmp ult i8 [[T2]], [[T0]] +; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[T0]], [[T2]] ; CHECK-NEXT: ret i1 [[R]] ; %t0 = shl i8 -1, %bits0 diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll index cf560d7f333f8395b89d35063764f4f5edde7232..e346330aa5b1e382988ccde4415340ad9742f7d4 100644 --- a/llvm/test/Transforms/InstCombine/known-bits.ll +++ b/llvm/test/Transforms/InstCombine/known-bits.ll @@ -48,4 +48,246 @@ define void @test_udiv(i8 %x) { ret void } +define i8 @test_cond(i8 %x) { +; CHECK-LABEL: @test_cond( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: br i1 [[CMP]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: ret i8 -4 +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp eq i8 %and, 0 + br i1 %cmp, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_inv(i8 %x) { +; CHECK-LABEL: @test_cond_inv( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[AND]], 0 +; CHECK-NEXT: call void @use(i1 [[CMP]]) +; CHECK-NEXT: br i1 [[CMP]], label [[EXIT:%.*]], label [[IF:%.*]] +; CHECK: if: +; CHECK-NEXT: ret i8 -4 +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp ne i8 %and, 0 + call void @use(i1 %cmp) + br i1 %cmp, label %exit, label %if + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_and(i8 %x, i1 %c) { +; CHECK-LABEL: @test_cond_and( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: [[COND:%.*]] = and i1 [[CMP]], [[C:%.*]] +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[OR1:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR1]] +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp eq i8 %and, 0 + %cond = and i1 %cmp, %c + br i1 %cond, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_and_commuted(i8 %x, i1 %c1, i1 %c2) { +; CHECK-LABEL: @test_cond_and_commuted( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: [[C3:%.*]] = and i1 [[C1:%.*]], [[C2:%.*]] +; CHECK-NEXT: [[COND:%.*]] = and i1 [[C3]], [[CMP]] +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[OR1:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR1]] +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp eq i8 %and, 0 + %c3 = and i1 %c1, %c2 + %cond = and i1 %c3, %cmp + br i1 %cond, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_logical_and(i8 %x, i1 %c) { +; CHECK-LABEL: @test_cond_logical_and( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: [[COND:%.*]] = select i1 [[CMP]], i1 [[C:%.*]], i1 false +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[OR1:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR1]] +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp eq i8 %and, 0 + %cond = select i1 %cmp, i1 %c, i1 false + br i1 %cond, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_or_invalid(i8 %x, i1 %c) { +; CHECK-LABEL: @test_cond_or_invalid( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: [[COND:%.*]] = or i1 [[CMP]], [[C:%.*]] +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[OR1:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR1]] +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp eq i8 %and, 0 + %cond = or i1 %cmp, %c + br i1 %cond, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_inv_or(i8 %x, i1 %c) { +; CHECK-LABEL: @test_cond_inv_or( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[AND]], 0 +; CHECK-NEXT: [[COND:%.*]] = or i1 [[CMP]], [[C:%.*]] +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[OR1:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR1]] +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp ne i8 %and, 0 + %cond = or i1 %cmp, %c + br i1 %cond, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_inv_logical_or(i8 %x, i1 %c) { +; CHECK-LABEL: @test_cond_inv_logical_or( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: [[COND:%.*]] = select i1 [[CMP_NOT]], i1 [[C:%.*]], i1 false +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[OR1:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR1]] +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp ne i8 %and, 0 + %cond = select i1 %cmp, i1 false, i1 %c + br i1 %cond, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_inv_and_invalid(i8 %x, i1 %c) { +; CHECK-LABEL: @test_cond_inv_and_invalid( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[AND]], 0 +; CHECK-NEXT: [[COND:%.*]] = and i1 [[CMP]], [[C:%.*]] +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[OR1:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR1]] +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp ne i8 %and, 0 + %cond = and i1 %cmp, %c + br i1 %cond, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +declare void @use(i1) declare void @sink(i8) diff --git a/llvm/test/Transforms/InstCombine/not.ll b/llvm/test/Transforms/InstCombine/not.ll index 847625af8b9c89695436d1d0d373b9403a585bf9..3b0e5b4412fbec05778b32f5d4a87b973da7340c 100644 --- a/llvm/test/Transforms/InstCombine/not.ll +++ b/llvm/test/Transforms/InstCombine/not.ll @@ -39,7 +39,7 @@ define i1 @invert_fcmp(float %X, float %Y) { define i1 @not_not_cmp(i32 %a, i32 %b) { ; CHECK-LABEL: @not_not_cmp( -; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[B:%.*]], [[A:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: ret i1 [[CMP]] ; %nota = xor i32 %a, -1 @@ -50,7 +50,7 @@ define i1 @not_not_cmp(i32 %a, i32 %b) { define <2 x i1> @not_not_cmp_vector(<2 x i32> %a, <2 x i32> %b) { ; CHECK-LABEL: @not_not_cmp_vector( -; CHECK-NEXT: [[CMP:%.*]] = icmp ugt <2 x i32> [[B:%.*]], [[A:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = icmp ult <2 x i32> [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: ret <2 x i1> [[CMP]] ; %nota = xor <2 x i32> %a, @@ -727,9 +727,8 @@ define i8 @bitcast_to_scalar_sext_bool_use2(<4 x i1> %b) { define i1 @invert_both_cmp_operands_add(i32 %a, i32 %b) { ; CHECK-LABEL: @invert_both_cmp_operands_add( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[NOT_A:%.*]] = xor i32 [[A:%.*]], -1 -; CHECK-NEXT: [[ADD:%.*]] = add i32 [[NOT_A]], [[B:%.*]] -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[ADD]], 0 +; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], -1 ; CHECK-NEXT: ret i1 [[CMP]] ; entry: @@ -742,9 +741,8 @@ entry: define i1 @invert_both_cmp_operands_sub(i32 %a, i32 %b) { ; CHECK-LABEL: @invert_both_cmp_operands_sub( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[NOT_A:%.*]] = xor i32 [[A:%.*]], -1 -; CHECK-NEXT: [[ADD:%.*]] = sub i32 [[NOT_A]], [[B:%.*]] -; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[ADD]], 42 +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i32 [[TMP0]], -43 ; CHECK-NEXT: ret i1 [[CMP]] ; entry: @@ -757,12 +755,9 @@ entry: define i1 @invert_both_cmp_operands_complex(i1 %x, i32 %a, i32 %b, i32 %c) { ; CHECK-LABEL: @invert_both_cmp_operands_complex( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[NOT_A:%.*]] = xor i32 [[A:%.*]], -1 -; CHECK-NEXT: [[NOT_B:%.*]] = xor i32 [[B:%.*]], -1 -; CHECK-NEXT: [[NOT_C:%.*]] = xor i32 [[C:%.*]], -1 -; CHECK-NEXT: [[ADD:%.*]] = add i32 [[NOT_A]], [[C]] -; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[X:%.*]], i32 [[ADD]], i32 [[NOT_B]] -; CHECK-NEXT: [[CMP:%.*]] = icmp sle i32 [[SELECT]], [[NOT_C]] +; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[A:%.*]], [[C:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[X:%.*]], i32 [[TMP0]], i32 [[B:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = icmp sge i32 [[TMP1]], [[C]] ; CHECK-NEXT: ret i1 [[CMP]] ; entry: diff --git a/llvm/test/Transforms/InstCombine/or.ll b/llvm/test/Transforms/InstCombine/or.ll index fd415682b67f32c2a52c660185db9c5f4c69fbd7..8c8aab2bcba6a80348a9f4b5815aa23a5be66dbe 100644 --- a/llvm/test/Transforms/InstCombine/or.ll +++ b/llvm/test/Transforms/InstCombine/or.ll @@ -1608,3 +1608,16 @@ define i8 @drop_disjoint(i8 %x) { %b = or disjoint i8 %a, 1 ret i8 %b } + +; Make sure we drop disjoint when combining the Ors. +define i32 @assoc_cast_assoc_disjoint(i16 %x) { +; CHECK-LABEL: @assoc_cast_assoc_disjoint( +; CHECK-NEXT: [[B:%.*]] = zext i16 [[X:%.*]] to i32 +; CHECK-NEXT: [[C:%.*]] = or i32 [[B]], 65537 +; CHECK-NEXT: ret i32 [[C]] +; + %a = or i16 %x, 1 + %b = zext i16 %a to i32 + %c = or disjoint i32 %b, 65536 + ret i32 %c +} diff --git a/llvm/test/Transforms/InstSimplify/returned.ll b/llvm/test/Transforms/InstSimplify/returned.ll index 94a98ac6cb05de2f39804f42865c0b080b605115..2da1052e7d4d8983d9e04e294cccc831f9337e32 100644 --- a/llvm/test/Transforms/InstSimplify/returned.ll +++ b/llvm/test/Transforms/InstSimplify/returned.ll @@ -25,6 +25,31 @@ define i1 @gep3() { ret i1 %equal } +define <8 x i1> @returned_vec_arg_casted(<2 x i32> %a) { +; CHECK-LABEL: @returned_vec_arg_casted( +; CHECK-NEXT: [[X:%.*]] = call <8 x i8> @passthru_8i8v_from_2i32v(<2 x i32> [[A:%.*]]) +; CHECK-NEXT: [[C:%.*]] = icmp slt <8 x i8> [[X]], zeroinitializer +; CHECK-NEXT: ret <8 x i1> [[C]] +; + %x = call <8 x i8> @passthru_8i8v_from_2i32v(<2 x i32> %a) + %C = icmp slt <8 x i8> %x, zeroinitializer + ret <8 x i1> %C +} + +define <8 x i1> @returned_vec_arg_casted2(<2 x i32> %a) { +; CHECK-LABEL: @returned_vec_arg_casted2( +; CHECK-NEXT: [[OR:%.*]] = or <2 x i32> [[A:%.*]], +; CHECK-NEXT: [[X:%.*]] = call <8 x i8> @passthru_8i8v_from_2i32v(<2 x i32> [[OR]]) +; CHECK-NEXT: [[C:%.*]] = icmp ne <8 x i8> [[X]], zeroinitializer +; CHECK-NEXT: ret <8 x i1> [[C]] +; + %or = or <2 x i32> %a, + %x = call <8 x i8> @passthru_8i8v_from_2i32v(<2 x i32> %or) + %C = icmp ne <8 x i8> %x, zeroinitializer + ret <8 x i1> %C +} + +declare <8 x i8> @passthru_8i8v_from_2i32v(<2 x i32> returned) declare ptr @func1(ptr returned) nounwind readnone willreturn declare ptr @func2(ptr returned) nounwind readnone willreturn diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles-inseltpoison.ll index dd08172be1b871326a891ccbcb395ffae3e8b2f9..22df002dd62cd3796b8450c683b1b56c17c407a7 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles-inseltpoison.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles.ll index 2e8a7cf42ac506bff6273f9b3fc3087a2ba6153b..399fa5298b7ccc1009e1d93f8a8014752390f8f7 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll index ab70d623470c8ee65f8e13b1963f10160042684d..224a0693bf218191a0b5e861409a66ace4a55744 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 ; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=NEON ; RUN: opt < %s -interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve -force-streaming-compatible-sve -S | FileCheck %s --check-prefix=SVE-FIXED +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=NEON +; RUN: opt < %s -passes=interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve -force-streaming-compatible-sve -S | FileCheck %s --check-prefix=SVE-FIXED target triple = "aarch64-linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user-inseltpoison.ll index db031ed12b7ec841acc23956ccba43d7defcc579..e48dc5d3051bfdcf87340e4b45742aa9c1451b7b 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user-inseltpoison.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll index af472630b951833640f3dea416969ae8c6b4a3f8..ea33590cb241bd4eeccc156c50ca66696fbcb979 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll index 77fbb8e7f2cad15f0509e42e669bbd289f1e1d89..14986d9eb85c5d0d92cad3c5ec2cc945d76a6c00 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll @@ -1,5 +1,7 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s -check-prefix=NEON ; RUN: opt < %s -mattr=-neon -interleaved-access -S | FileCheck %s -check-prefix=NO_NEON +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s -check-prefix=NEON +; RUN: opt < %s -mattr=-neon -passes=interleaved-access -S | FileCheck %s -check-prefix=NO_NEON target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll index 77fbb8e7f2cad15f0509e42e669bbd289f1e1d89..14986d9eb85c5d0d92cad3c5ec2cc945d76a6c00 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll @@ -1,5 +1,7 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s -check-prefix=NEON ; RUN: opt < %s -mattr=-neon -interleaved-access -S | FileCheck %s -check-prefix=NO_NEON +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s -check-prefix=NEON +; RUN: opt < %s -mattr=-neon -passes=interleaved-access -S | FileCheck %s -check-prefix=NO_NEON target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll index c04464b2ca9df22e32be838995a1dac4e71da5e8..6353bf10d57c44d58b0d607d59a98267b8f4eca4 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target triple = "aarch64-linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll index 94f63e5921f8b8ada77981dd940321c56e39973c..feb22aa1a37635de7728d979df23de200ed3095a 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target triple = "aarch64-linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user-inseltpoison.ll index 3ea00de0dcb1545119d433f00e5839b9634ec482..77b376b3580ca33864a479c2bdbe61d421c68144 100644 --- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user-inseltpoison.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64" target triple = "arm---eabi" diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll index 0c56f71cc3693a925df6e12fe17b0b697839248b..8934941678723e502507cec3affdeb91b387c3ee 100644 --- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll +++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64" target triple = "arm---eabi" diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll index 9ea1c5e94a98358e5e790d032d1d96182f0eb6ec..aed843723189b6645d06bdcefd90cd7d2a6f7810 100644 --- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll @@ -2,6 +2,9 @@ ; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON ; RUN: opt < %s -mattr=+mve.fp -interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE ; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE +; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON +; RUN: opt < %s -mattr=+mve.fp -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64" target triple = "arm---eabi" diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll index c84d759ae0fb32f9170c3e60e1f8a8b08915558e..8123ea5bbe9f1f8971c8670ee8216fced4d2b2a4 100644 --- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll +++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll @@ -2,6 +2,9 @@ ; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON ; RUN: opt < %s -mattr=+mve.fp -interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE ; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE +; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON +; RUN: opt < %s -mattr=+mve.fp -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64" target triple = "arm---eabi" diff --git a/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll index 736a7ce4ecf0cbe5a1700889beb6b50cc3f56c3e..ea2d1ca722ca0d853dad3582ee0a05ead3ebe219 100644 --- a/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll +++ b/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=riscv32 -mattr=+v -interleaved-access -S | FileCheck %s --check-prefix=RV32 ; RUN: opt < %s -mtriple=riscv64 -mattr=+v -interleaved-access -S | FileCheck %s --check-prefix=RV64 +; RUN: opt < %s -mtriple=riscv32 -mattr=+v -passes=interleaved-access -S | FileCheck %s --check-prefix=RV32 +; RUN: opt < %s -mtriple=riscv64 -mattr=+v -passes=interleaved-access -S | FileCheck %s --check-prefix=RV64 define void @load_factor2(ptr %ptr) { ; RV32-LABEL: @load_factor2( diff --git a/llvm/test/Transforms/InterleavedAccess/RISCV/zve32x.ll b/llvm/test/Transforms/InterleavedAccess/RISCV/zve32x.ll index e7b7ec052f459391fcce0d42ca40b2daed867809..ac3cab6638b8c82d6ddbb9d95891498f9d8a08dd 100644 --- a/llvm/test/Transforms/InterleavedAccess/RISCV/zve32x.ll +++ b/llvm/test/Transforms/InterleavedAccess/RISCV/zve32x.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=riscv64 -mattr=+zve32x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVE32X ; RUN: opt < %s -mtriple=riscv64 -mattr=+zve64x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVE64X +; RUN: opt < %s -mtriple=riscv64 -mattr=+zve32x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVE32X +; RUN: opt < %s -mtriple=riscv64 -mattr=+zve64x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVE64X define <4 x i1> @load_large_vector(ptr %p) { ; ZVE32X-LABEL: @load_large_vector( diff --git a/llvm/test/Transforms/InterleavedAccess/RISCV/zvl32b.ll b/llvm/test/Transforms/InterleavedAccess/RISCV/zvl32b.ll index 9c896796760ff60bea2645cf037efb77fb92a2c8..a94e6a70e79e684434f7e83cf2ddd3721cc62685 100644 --- a/llvm/test/Transforms/InterleavedAccess/RISCV/zvl32b.ll +++ b/llvm/test/Transforms/InterleavedAccess/RISCV/zvl32b.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl32b -interleaved-access -S | FileCheck %s -check-prefix=ZVL32B ; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVL128B +; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl32b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVL32B +; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVL128B ; Make sure that we don't lower interleaved loads that won't fit into the minimum vlen diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleave-load-extract-shuffle-changes.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleave-load-extract-shuffle-changes.ll index a4b4e3748b46360b6343590efbd2da6a3c91c799..167d282edb3e72fa270f5d178158ed39b91614d5 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleave-load-extract-shuffle-changes.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleave-load-extract-shuffle-changes.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -interleaved-access -S %s | FileCheck %s +; RUN: opt -passes=interleaved-access -S %s | FileCheck %s target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.15.0" diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll index b4128dda7cee77208145a6069e75648e719a29a7..13076c7ffec9cab8c35844f2668dcaf68c5de4f1 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -passes=interleaved-access -S | FileCheck %s ; This file tests the function `llvm::lowerInterleavedLoad/Store`. diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll index ec905a999dca19bdc70a30f530bac6fab890d04a..6972afe486bd066b2f5c2848ab3763031c8fe774 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -passes=interleaved-access -S | FileCheck %s ; This file tests the function `llvm::lowerInterleavedLoad/Store`. diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad-inseltpoison.ll index 9b38f2dfbabd84702d96f29a85625d92bd69918d..327a1d2c6a66c7859d9fd1786720239bc566a671 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad-inseltpoison.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -interleaved-access -S | FileCheck %s ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx512f -mattr=+avx512bw -mattr=+avx512vl -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -passes=interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx512f -mattr=+avx512bw -mattr=+avx512vl -passes=interleaved-access -S | FileCheck %s define <32 x i8> @interleaved_load_vf32_i8_stride3(ptr %ptr){ ; CHECK-LABEL: @interleaved_load_vf32_i8_stride3( diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll index 86c18627625810b039df3c375625f319b14942fc..68d8b947551789791e4400e81daf8ab04799763e 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -interleaved-access -S | FileCheck %s ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx512f -mattr=+avx512bw -mattr=+avx512vl -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -passes=interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx512f -mattr=+avx512bw -mattr=+avx512vl -passes=interleaved-access -S | FileCheck %s define <32 x i8> @interleaved_load_vf32_i8_stride3(ptr %ptr){ ; CHECK-LABEL: @interleaved_load_vf32_i8_stride3( diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll index 301abda5708cf5b0f57d28ce83968fecc45243fe..d1eea7a0d1bfd6a65b140666db227b31c73d586f 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -passes=interleaved-access -S | FileCheck %s define void @interleaved_store_vf32_i8_stride4(<32 x i8> %x1, <32 x i8> %x2, <32 x i8> %x3, <32 x i8> %x4, ptr %p) { ; CHECK-LABEL: @interleaved_store_vf32_i8_stride4( diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll index 75595463c740484ec73f7562e0776e2498533d5d..3e7bc4130a8cdd48ecc3de289bb3a1809452d15c 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -passes=interleaved-access -S | FileCheck %s define void @interleaved_store_vf32_i8_stride4(<32 x i8> %x1, <32 x i8> %x2, <32 x i8> %x3, <32 x i8> %x4, ptr %p) { ; CHECK-LABEL: @interleaved_store_vf32_i8_stride4( diff --git a/llvm/test/Transforms/LICM/pr64897.ll b/llvm/test/Transforms/LICM/pr64897.ll index 1ce78f15d797359627465a488f6891c22c3557e7..12b11eb6912b0cbf7b4a11eea0b70219a59f83d6 100644 --- a/llvm/test/Transforms/LICM/pr64897.ll +++ b/llvm/test/Transforms/LICM/pr64897.ll @@ -1,7 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 ; RUN: opt -S -passes=licm < %s | FileCheck %s -; FIXME: This is a miscompile. define void @test(i1 %c, i8 %x) { ; CHECK-LABEL: define void @test( ; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]]) { @@ -10,17 +9,16 @@ define void @test(i1 %c, i8 %x) { ; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 8 ; CHECK-NEXT: [[P_COPY:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 8 ; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 12 -; CHECK-NEXT: [[P2_PROMOTED:%.*]] = load i8, ptr [[P2]], align 1 ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[TMP0:%.*]] = phi i8 [ 0, [[LOOP]] ], [ [[P2_PROMOTED]], [[START:%.*]] ] ; CHECK-NEXT: store i32 286331153, ptr [[P]], align 4 ; CHECK-NEXT: store i32 34, ptr [[P_COPY]], align 4 ; CHECK-NEXT: store i64 3689348814741910323, ptr [[P_COPY]], align 4 -; CHECK-NEXT: call void @use(i8 [[TMP0]]) +; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[P2]], align 1 +; CHECK-NEXT: call void @use(i8 [[VAL]]) +; CHECK-NEXT: store i8 0, ptr [[P2]], align 1 ; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]] ; CHECK: exit: -; CHECK-NEXT: store i8 0, ptr [[P2]], align 1 ; CHECK-NEXT: ret void ; start: diff --git a/llvm/test/Transforms/LoopPredication/basic.ll b/llvm/test/Transforms/LoopPredication/basic.ll index 2a99963ad0786c459948b05136f53af8e407e0fa..27c8bc99c407e0cbbd22cace6143faaf2b656b03 100644 --- a/llvm/test/Transforms/LoopPredication/basic.ll +++ b/llvm/test/Transforms/LoopPredication/basic.ll @@ -1681,7 +1681,7 @@ define i32 @ne_latch_zext(ptr %array, i32 %length, i16 %n16) { ; CHECK-LABEL: @ne_latch_zext( ; CHECK-NEXT: loop.preheader: ; CHECK-NEXT: [[N:%.*]] = zext i16 [[N16:%.*]] to i32 -; CHECK-NEXT: [[NPLUS1:%.*]] = add i32 [[N]], 1 +; CHECK-NEXT: [[NPLUS1:%.*]] = add nuw nsw i32 [[N]], 1 ; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i32 [[NPLUS1]], [[LENGTH:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]] ; CHECK-NEXT: [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]] diff --git a/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll b/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll index c9b66896eac7859a64c3f68fbc2bd56184e7f0c8..0203abe69ac299f0ffa98fe00eac02a6b888a313 100644 --- a/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll +++ b/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll @@ -453,7 +453,7 @@ define void @non_branch_terminator(ptr %a) { ; CHECK-NEXT: [[UGLYGEP2]] = getelementptr i8, ptr [[LSR_IV1]], i64 4 ; CHECK-NEXT: [[LSR_IV_NEXT3]] = add nsw i64 [[LSR_IV2]], -1 ; CHECK-NEXT: switch i64 [[LSR_IV2]], label [[FOR_BODY]] [ -; CHECK-NEXT: i64 0, label [[FOR_END:%.*]] +; CHECK-NEXT: i64 0, label [[FOR_END:%.*]] ; CHECK-NEXT: ] ; CHECK: for.end: ; CHECK-NEXT: ret void @@ -473,3 +473,117 @@ for.body: ; preds = %for.body, %entry for.end: ; preds = %for.body ret void } + +define void @expensive_expand_short_tc(ptr %a, i32 %offset, i32 %n) { +; CHECK-LABEL: @expensive_expand_short_tc( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 84 +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 +; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 +; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[OFFSET:%.*]] to i64 +; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP2]], [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], 84 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[LSR_IV1:%.*]] = phi ptr [ [[UGLYGEP2:%.*]], [[FOR_BODY]] ], [ [[UGLYGEP]], [[ENTRY:%.*]] ] +; CHECK-NEXT: store i32 1, ptr [[LSR_IV1]], align 4 +; CHECK-NEXT: [[UGLYGEP2]] = getelementptr i8, ptr [[LSR_IV1]], i32 [[OFFSET]] +; CHECK-NEXT: [[LSR_FOLD_TERM_COND_REPLACED_TERM_COND:%.*]] = icmp eq ptr [[UGLYGEP2]], [[SCEVGEP]] +; CHECK-NEXT: br i1 [[LSR_FOLD_TERM_COND_REPLACED_TERM_COND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !prof [[PROF0:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: ret void +; +entry: + %uglygep = getelementptr i8, ptr %a, i64 84 + br label %for.body + +for.body: ; preds = %for.body, %entry + %lsr.iv1 = phi ptr [ %uglygep2, %for.body ], [ %uglygep, %entry ] + %lsr.iv = phi i32 [ %lsr.iv.next, %for.body ], [ 0, %entry ] + store i32 1, ptr %lsr.iv1, align 4 + %lsr.iv.next = add nsw i32 %lsr.iv, 1 + %uglygep2 = getelementptr i8, ptr %lsr.iv1, i32 %offset + %exitcond.not = icmp eq i32 %lsr.iv.next, %n + br i1 %exitcond.not, label %for.end, label %for.body, !prof !{!"branch_weights", i32 1, i32 3} + +for.end: ; preds = %for.body + ret void +} + +define void @expensive_expand_long_tc(ptr %a, i32 %offset, i32 %n) { +; CHECK-LABEL: @expensive_expand_long_tc( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 84 +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 +; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 +; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[OFFSET:%.*]] to i64 +; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP2]], [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], 84 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[LSR_IV1:%.*]] = phi ptr [ [[UGLYGEP2:%.*]], [[FOR_BODY]] ], [ [[UGLYGEP]], [[ENTRY:%.*]] ] +; CHECK-NEXT: store i32 1, ptr [[LSR_IV1]], align 4 +; CHECK-NEXT: [[UGLYGEP2]] = getelementptr i8, ptr [[LSR_IV1]], i32 [[OFFSET]] +; CHECK-NEXT: [[LSR_FOLD_TERM_COND_REPLACED_TERM_COND:%.*]] = icmp eq ptr [[UGLYGEP2]], [[SCEVGEP]] +; CHECK-NEXT: br i1 [[LSR_FOLD_TERM_COND_REPLACED_TERM_COND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !prof [[PROF1:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: ret void +; +entry: + %uglygep = getelementptr i8, ptr %a, i64 84 + br label %for.body + +for.body: ; preds = %for.body, %entry + %lsr.iv1 = phi ptr [ %uglygep2, %for.body ], [ %uglygep, %entry ] + %lsr.iv = phi i32 [ %lsr.iv.next, %for.body ], [ 0, %entry ] + store i32 1, ptr %lsr.iv1, align 4 + %lsr.iv.next = add nsw i32 %lsr.iv, 1 + %uglygep2 = getelementptr i8, ptr %lsr.iv1, i32 %offset + %exitcond.not = icmp eq i32 %lsr.iv.next, %n + br i1 %exitcond.not, label %for.end, label %for.body, !prof !{!"branch_weights", i32 1, i32 300} + +for.end: ; preds = %for.body + ret void +} + +define void @expensive_expand_unknown_tc(ptr %a, i32 %offset, i32 %n) { +; CHECK-LABEL: @expensive_expand_unknown_tc( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 84 +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 +; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 +; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[OFFSET:%.*]] to i64 +; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP2]], [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], 84 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[LSR_IV1:%.*]] = phi ptr [ [[UGLYGEP2:%.*]], [[FOR_BODY]] ], [ [[UGLYGEP]], [[ENTRY:%.*]] ] +; CHECK-NEXT: store i32 1, ptr [[LSR_IV1]], align 4 +; CHECK-NEXT: [[UGLYGEP2]] = getelementptr i8, ptr [[LSR_IV1]], i32 [[OFFSET]] +; CHECK-NEXT: [[LSR_FOLD_TERM_COND_REPLACED_TERM_COND:%.*]] = icmp eq ptr [[UGLYGEP2]], [[SCEVGEP]] +; CHECK-NEXT: br i1 [[LSR_FOLD_TERM_COND_REPLACED_TERM_COND]], label [[FOR_END:%.*]], label [[FOR_BODY]] +; CHECK: for.end: +; CHECK-NEXT: ret void +; +entry: + %uglygep = getelementptr i8, ptr %a, i64 84 + br label %for.body + +for.body: ; preds = %for.body, %entry + %lsr.iv1 = phi ptr [ %uglygep2, %for.body ], [ %uglygep, %entry ] + %lsr.iv = phi i32 [ %lsr.iv.next, %for.body ], [ 0, %entry ] + store i32 1, ptr %lsr.iv1, align 4 + %lsr.iv.next = add nsw i32 %lsr.iv, 1 + %uglygep2 = getelementptr i8, ptr %lsr.iv1, i32 %offset + %exitcond.not = icmp eq i32 %lsr.iv.next, %n + br i1 %exitcond.not, label %for.end, label %for.body + +for.end: ; preds = %for.body + ret void +} diff --git a/llvm/test/Transforms/LoopUnroll/loop-branch-folding.ll b/llvm/test/Transforms/LoopUnroll/loop-branch-folding.ll new file mode 100644 index 0000000000000000000000000000000000000000..f0a6c0a954f6aedc39ec47868b715d8d0fe44db9 --- /dev/null +++ b/llvm/test/Transforms/LoopUnroll/loop-branch-folding.ll @@ -0,0 +1,936 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -S -passes=simplifycfg | FileCheck %s --check-prefixes=CHECK-CFG +; RUN: opt < %s -S -passes=simplifycfg,loop-unroll --unroll-max-upperbound=17 | FileCheck %s --check-prefixes=CHECK-UNROLL + +; This test designed to check: +; We can still unroll loop with 'pragma unroll' if loop count(trip count) was destroyed by previous optimization. +; For exmaple, in following test, loop condition "Dim < 16" was 'merged' with "Dim == Dims" in folding branches +; at simplifycfg. But if custumer mark the loop with "#pragma unroll", we can still successfully unroll it under +; unroll-max-upperbound. +; +; __device__ void func(int Idx, int *Arr[], int Dims, int *Out) { +; #pragma unroll +; for (int Dim = 0; Dim < 16; ++Dim) { +; if (Dim == Dims) { +; break; +; } +; int divmod = Arr[Dim][Idx]; +; Idx = divmod + 1; +; +; for (int arg = 0; arg < 4; arg++) { +; Out[arg] += Arr[Dim][arg]; +; bar(); +; } +; } +; } + +define void @func(i32 noundef %Idx, ptr noundef %Arr, i32 noundef %Dims, ptr noundef %Out) { +; CHECK-CFG-LABEL: define void @func( +; CHECK-CFG-SAME: i32 noundef [[IDX:%.*]], ptr noundef [[ARR:%.*]], i32 noundef [[DIMS:%.*]], ptr noundef [[OUT:%.*]]) { +; CHECK-CFG-NEXT: entry: +; CHECK-CFG-NEXT: br label [[FOR_COND:%.*]] +; CHECK-CFG: for.cond: +; CHECK-CFG-NEXT: [[DIM_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC16:%.*]], [[FOR_COND_CLEANUP6:%.*]] ] +; CHECK-CFG-NEXT: [[IDX_ADDR_0:%.*]] = phi i32 [ [[IDX]], [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_COND_CLEANUP6]] ] +; CHECK-CFG-NEXT: [[CMP:%.*]] = icmp sge i32 [[DIM_0]], 16 +; CHECK-CFG-NEXT: [[CMP1:%.*]] = icmp eq i32 [[DIM_0]], [[DIMS]] +; CHECK-CFG-NEXT: [[OR_COND:%.*]] = or i1 [[CMP]], [[CMP1]] +; CHECK-CFG-NEXT: br i1 [[OR_COND]], label [[CLEANUP:%.*]], label [[IF_END:%.*]] +; CHECK-CFG: if.end: +; CHECK-CFG-NEXT: [[IDXPROM:%.*]] = sext i32 [[DIM_0]] to i64 +; CHECK-CFG-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 [[IDXPROM]] +; CHECK-CFG-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8 +; CHECK-CFG-NEXT: [[IDXPROM2:%.*]] = sext i32 [[IDX_ADDR_0]] to i64 +; CHECK-CFG-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 [[IDXPROM2]] +; CHECK-CFG-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4 +; CHECK-CFG-NEXT: [[ADD]] = add nsw i32 [[TMP1]], 1 +; CHECK-CFG-NEXT: br label [[FOR_COND4:%.*]] +; CHECK-CFG: for.cond4: +; CHECK-CFG-NEXT: [[ARG_0:%.*]] = phi i32 [ 0, [[IF_END]] ], [ [[INC:%.*]], [[FOR_BODY7:%.*]] ] +; CHECK-CFG-NEXT: [[CMP5:%.*]] = icmp slt i32 [[ARG_0]], 4 +; CHECK-CFG-NEXT: br i1 [[CMP5]], label [[FOR_BODY7]], label [[FOR_COND_CLEANUP6]] +; CHECK-CFG: for.cond.cleanup6: +; CHECK-CFG-NEXT: [[INC16]] = add nsw i32 [[DIM_0]], 1 +; CHECK-CFG-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-CFG: for.body7: +; CHECK-CFG-NEXT: [[TMP2:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8 +; CHECK-CFG-NEXT: [[IDXPROM10:%.*]] = sext i32 [[ARG_0]] to i64 +; CHECK-CFG-NEXT: [[ARRAYIDX11:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 [[IDXPROM10]] +; CHECK-CFG-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX11]], align 4 +; CHECK-CFG-NEXT: [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[IDXPROM10]] +; CHECK-CFG-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX13]], align 4 +; CHECK-CFG-NEXT: [[ADD14:%.*]] = add nsw i32 [[TMP4]], [[TMP3]] +; CHECK-CFG-NEXT: store i32 [[ADD14]], ptr [[ARRAYIDX13]], align 4 +; CHECK-CFG-NEXT: call void @_Z3barv() +; CHECK-CFG-NEXT: [[INC]] = add nsw i32 [[ARG_0]], 1 +; CHECK-CFG-NEXT: br label [[FOR_COND4]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-CFG: cleanup: +; CHECK-CFG-NEXT: ret void +; +; CHECK-UNROLL-LABEL: define void @func( +; CHECK-UNROLL-SAME: i32 noundef [[IDX:%.*]], ptr noundef [[ARR:%.*]], i32 noundef [[DIMS:%.*]], ptr noundef [[OUT:%.*]]) { +; CHECK-UNROLL-NEXT: entry: +; CHECK-UNROLL-NEXT: br label [[FOR_COND:%.*]] +; CHECK-UNROLL: for.cond: +; CHECK-UNROLL-NEXT: [[CMP1:%.*]] = icmp eq i32 0, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1]], label [[CLEANUP:%.*]], label [[IF_END:%.*]] +; CHECK-UNROLL: if.end: +; CHECK-UNROLL-NEXT: br label [[FOR_COND4:%.*]] +; CHECK-UNROLL: for.cond4: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7:%.*]] +; CHECK-UNROLL: for.cond.cleanup6: +; CHECK-UNROLL-NEXT: [[CMP1_1:%.*]] = icmp eq i32 1, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_1]], label [[CLEANUP]], label [[IF_END_1:%.*]] +; CHECK-UNROLL: if.end.1: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 1 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_1:%.*]] +; CHECK-UNROLL: for.cond4.1: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_12:%.*]] +; CHECK-UNROLL: for.body7.12: +; CHECK-UNROLL-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX_1]], align 8 +; CHECK-UNROLL-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4 +; CHECK-UNROLL-NEXT: [[TMP2:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_11:%.*]] = add nsw i32 [[TMP2]], [[TMP1]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_11]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_1:%.*]] +; CHECK-UNROLL: for.body7.1.1: +; CHECK-UNROLL-NEXT: [[TMP3:%.*]] = load ptr, ptr [[ARRAYIDX_1]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_1:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX11_1_1]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX13_1_1]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_1:%.*]] = add nsw i32 [[TMP5]], [[TMP4]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_1]], ptr [[ARRAYIDX13_1_1]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_1:%.*]] +; CHECK-UNROLL: for.body7.2.1: +; CHECK-UNROLL-NEXT: [[TMP6:%.*]] = load ptr, ptr [[ARRAYIDX_1]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_1:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX11_2_1]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX13_2_1]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_1:%.*]] = add nsw i32 [[TMP8]], [[TMP7]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_1]], ptr [[ARRAYIDX13_2_1]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_1:%.*]] +; CHECK-UNROLL: for.body7.3.1: +; CHECK-UNROLL-NEXT: [[TMP9:%.*]] = load ptr, ptr [[ARRAYIDX_1]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_1:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX11_3_1]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX13_3_1]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_1:%.*]] = add nsw i32 [[TMP11]], [[TMP10]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_1]], ptr [[ARRAYIDX13_3_1]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4:%.*]], label [[FOR_COND_CLEANUP6_1:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.1: +; CHECK-UNROLL-NEXT: [[CMP1_2:%.*]] = icmp eq i32 2, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_2]], label [[CLEANUP]], label [[IF_END_2:%.*]] +; CHECK-UNROLL: if.end.2: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 2 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_2:%.*]] +; CHECK-UNROLL: for.cond4.2: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_24:%.*]] +; CHECK-UNROLL: for.body7.24: +; CHECK-UNROLL-NEXT: [[TMP12:%.*]] = load ptr, ptr [[ARRAYIDX_2]], align 8 +; CHECK-UNROLL-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4 +; CHECK-UNROLL-NEXT: [[TMP14:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_23:%.*]] = add nsw i32 [[TMP14]], [[TMP13]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_23]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_2:%.*]] +; CHECK-UNROLL: for.body7.1.2: +; CHECK-UNROLL-NEXT: [[TMP15:%.*]] = load ptr, ptr [[ARRAYIDX_2]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_2:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX11_1_2]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX13_1_2]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_2:%.*]] = add nsw i32 [[TMP17]], [[TMP16]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_2]], ptr [[ARRAYIDX13_1_2]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_2:%.*]] +; CHECK-UNROLL: for.body7.2.2: +; CHECK-UNROLL-NEXT: [[TMP18:%.*]] = load ptr, ptr [[ARRAYIDX_2]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_2:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX11_2_2]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX13_2_2]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_2:%.*]] = add nsw i32 [[TMP20]], [[TMP19]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_2]], ptr [[ARRAYIDX13_2_2]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_2:%.*]] +; CHECK-UNROLL: for.body7.3.2: +; CHECK-UNROLL-NEXT: [[TMP21:%.*]] = load ptr, ptr [[ARRAYIDX_2]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_2:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP22:%.*]] = load i32, ptr [[ARRAYIDX11_3_2]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP23:%.*]] = load i32, ptr [[ARRAYIDX13_3_2]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_2:%.*]] = add nsw i32 [[TMP23]], [[TMP22]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_2]], ptr [[ARRAYIDX13_3_2]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_2:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.2: +; CHECK-UNROLL-NEXT: [[CMP1_3:%.*]] = icmp eq i32 3, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_3]], label [[CLEANUP]], label [[IF_END_3:%.*]] +; CHECK-UNROLL: if.end.3: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 3 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_3:%.*]] +; CHECK-UNROLL: for.cond4.3: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_36:%.*]] +; CHECK-UNROLL: for.body7.36: +; CHECK-UNROLL-NEXT: [[TMP24:%.*]] = load ptr, ptr [[ARRAYIDX_3]], align 8 +; CHECK-UNROLL-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP24]], align 4 +; CHECK-UNROLL-NEXT: [[TMP26:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_35:%.*]] = add nsw i32 [[TMP26]], [[TMP25]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_35]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_3:%.*]] +; CHECK-UNROLL: for.body7.1.3: +; CHECK-UNROLL-NEXT: [[TMP27:%.*]] = load ptr, ptr [[ARRAYIDX_3]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_3:%.*]] = getelementptr inbounds i32, ptr [[TMP27]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP28:%.*]] = load i32, ptr [[ARRAYIDX11_1_3]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP29:%.*]] = load i32, ptr [[ARRAYIDX13_1_3]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_3:%.*]] = add nsw i32 [[TMP29]], [[TMP28]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_3]], ptr [[ARRAYIDX13_1_3]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_3:%.*]] +; CHECK-UNROLL: for.body7.2.3: +; CHECK-UNROLL-NEXT: [[TMP30:%.*]] = load ptr, ptr [[ARRAYIDX_3]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_3:%.*]] = getelementptr inbounds i32, ptr [[TMP30]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP31:%.*]] = load i32, ptr [[ARRAYIDX11_2_3]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP32:%.*]] = load i32, ptr [[ARRAYIDX13_2_3]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_3:%.*]] = add nsw i32 [[TMP32]], [[TMP31]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_3]], ptr [[ARRAYIDX13_2_3]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_3:%.*]] +; CHECK-UNROLL: for.body7.3.3: +; CHECK-UNROLL-NEXT: [[TMP33:%.*]] = load ptr, ptr [[ARRAYIDX_3]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_3:%.*]] = getelementptr inbounds i32, ptr [[TMP33]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP34:%.*]] = load i32, ptr [[ARRAYIDX11_3_3]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP35:%.*]] = load i32, ptr [[ARRAYIDX13_3_3]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_3:%.*]] = add nsw i32 [[TMP35]], [[TMP34]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_3]], ptr [[ARRAYIDX13_3_3]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_3:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.3: +; CHECK-UNROLL-NEXT: [[CMP1_4:%.*]] = icmp eq i32 4, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_4]], label [[CLEANUP]], label [[IF_END_4:%.*]] +; CHECK-UNROLL: if.end.4: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 4 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_4:%.*]] +; CHECK-UNROLL: for.cond4.4: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_48:%.*]] +; CHECK-UNROLL: for.body7.48: +; CHECK-UNROLL-NEXT: [[TMP36:%.*]] = load ptr, ptr [[ARRAYIDX_4]], align 8 +; CHECK-UNROLL-NEXT: [[TMP37:%.*]] = load i32, ptr [[TMP36]], align 4 +; CHECK-UNROLL-NEXT: [[TMP38:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_47:%.*]] = add nsw i32 [[TMP38]], [[TMP37]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_47]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_4:%.*]] +; CHECK-UNROLL: for.body7.1.4: +; CHECK-UNROLL-NEXT: [[TMP39:%.*]] = load ptr, ptr [[ARRAYIDX_4]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_4:%.*]] = getelementptr inbounds i32, ptr [[TMP39]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP40:%.*]] = load i32, ptr [[ARRAYIDX11_1_4]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_4:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP41:%.*]] = load i32, ptr [[ARRAYIDX13_1_4]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_4:%.*]] = add nsw i32 [[TMP41]], [[TMP40]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_4]], ptr [[ARRAYIDX13_1_4]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_4:%.*]] +; CHECK-UNROLL: for.body7.2.4: +; CHECK-UNROLL-NEXT: [[TMP42:%.*]] = load ptr, ptr [[ARRAYIDX_4]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_4:%.*]] = getelementptr inbounds i32, ptr [[TMP42]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP43:%.*]] = load i32, ptr [[ARRAYIDX11_2_4]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_4:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP44:%.*]] = load i32, ptr [[ARRAYIDX13_2_4]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_4:%.*]] = add nsw i32 [[TMP44]], [[TMP43]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_4]], ptr [[ARRAYIDX13_2_4]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_4:%.*]] +; CHECK-UNROLL: for.body7.3.4: +; CHECK-UNROLL-NEXT: [[TMP45:%.*]] = load ptr, ptr [[ARRAYIDX_4]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_4:%.*]] = getelementptr inbounds i32, ptr [[TMP45]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP46:%.*]] = load i32, ptr [[ARRAYIDX11_3_4]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_4:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP47:%.*]] = load i32, ptr [[ARRAYIDX13_3_4]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_4:%.*]] = add nsw i32 [[TMP47]], [[TMP46]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_4]], ptr [[ARRAYIDX13_3_4]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_4:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.4: +; CHECK-UNROLL-NEXT: [[CMP1_5:%.*]] = icmp eq i32 5, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_5]], label [[CLEANUP]], label [[IF_END_5:%.*]] +; CHECK-UNROLL: if.end.5: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 5 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_5:%.*]] +; CHECK-UNROLL: for.cond4.5: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_5:%.*]] +; CHECK-UNROLL: for.body7.5: +; CHECK-UNROLL-NEXT: [[TMP48:%.*]] = load ptr, ptr [[ARRAYIDX_5]], align 8 +; CHECK-UNROLL-NEXT: [[TMP49:%.*]] = load i32, ptr [[TMP48]], align 4 +; CHECK-UNROLL-NEXT: [[TMP50:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_5:%.*]] = add nsw i32 [[TMP50]], [[TMP49]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_5]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_5:%.*]] +; CHECK-UNROLL: for.body7.1.5: +; CHECK-UNROLL-NEXT: [[TMP51:%.*]] = load ptr, ptr [[ARRAYIDX_5]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_5:%.*]] = getelementptr inbounds i32, ptr [[TMP51]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP52:%.*]] = load i32, ptr [[ARRAYIDX11_1_5]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_5:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP53:%.*]] = load i32, ptr [[ARRAYIDX13_1_5]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_5:%.*]] = add nsw i32 [[TMP53]], [[TMP52]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_5]], ptr [[ARRAYIDX13_1_5]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_5:%.*]] +; CHECK-UNROLL: for.body7.2.5: +; CHECK-UNROLL-NEXT: [[TMP54:%.*]] = load ptr, ptr [[ARRAYIDX_5]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_5:%.*]] = getelementptr inbounds i32, ptr [[TMP54]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP55:%.*]] = load i32, ptr [[ARRAYIDX11_2_5]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_5:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP56:%.*]] = load i32, ptr [[ARRAYIDX13_2_5]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_5:%.*]] = add nsw i32 [[TMP56]], [[TMP55]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_5]], ptr [[ARRAYIDX13_2_5]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_5:%.*]] +; CHECK-UNROLL: for.body7.3.5: +; CHECK-UNROLL-NEXT: [[TMP57:%.*]] = load ptr, ptr [[ARRAYIDX_5]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_5:%.*]] = getelementptr inbounds i32, ptr [[TMP57]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP58:%.*]] = load i32, ptr [[ARRAYIDX11_3_5]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_5:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP59:%.*]] = load i32, ptr [[ARRAYIDX13_3_5]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_5:%.*]] = add nsw i32 [[TMP59]], [[TMP58]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_5]], ptr [[ARRAYIDX13_3_5]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_5:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.5: +; CHECK-UNROLL-NEXT: [[CMP1_6:%.*]] = icmp eq i32 6, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_6]], label [[CLEANUP]], label [[IF_END_6:%.*]] +; CHECK-UNROLL: if.end.6: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 6 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_6:%.*]] +; CHECK-UNROLL: for.cond4.6: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_6:%.*]] +; CHECK-UNROLL: for.body7.6: +; CHECK-UNROLL-NEXT: [[TMP60:%.*]] = load ptr, ptr [[ARRAYIDX_6]], align 8 +; CHECK-UNROLL-NEXT: [[TMP61:%.*]] = load i32, ptr [[TMP60]], align 4 +; CHECK-UNROLL-NEXT: [[TMP62:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_6:%.*]] = add nsw i32 [[TMP62]], [[TMP61]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_6]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_6:%.*]] +; CHECK-UNROLL: for.body7.1.6: +; CHECK-UNROLL-NEXT: [[TMP63:%.*]] = load ptr, ptr [[ARRAYIDX_6]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_6:%.*]] = getelementptr inbounds i32, ptr [[TMP63]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP64:%.*]] = load i32, ptr [[ARRAYIDX11_1_6]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_6:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP65:%.*]] = load i32, ptr [[ARRAYIDX13_1_6]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_6:%.*]] = add nsw i32 [[TMP65]], [[TMP64]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_6]], ptr [[ARRAYIDX13_1_6]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_6:%.*]] +; CHECK-UNROLL: for.body7.2.6: +; CHECK-UNROLL-NEXT: [[TMP66:%.*]] = load ptr, ptr [[ARRAYIDX_6]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_6:%.*]] = getelementptr inbounds i32, ptr [[TMP66]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP67:%.*]] = load i32, ptr [[ARRAYIDX11_2_6]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_6:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP68:%.*]] = load i32, ptr [[ARRAYIDX13_2_6]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_6:%.*]] = add nsw i32 [[TMP68]], [[TMP67]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_6]], ptr [[ARRAYIDX13_2_6]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_6:%.*]] +; CHECK-UNROLL: for.body7.3.6: +; CHECK-UNROLL-NEXT: [[TMP69:%.*]] = load ptr, ptr [[ARRAYIDX_6]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_6:%.*]] = getelementptr inbounds i32, ptr [[TMP69]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP70:%.*]] = load i32, ptr [[ARRAYIDX11_3_6]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_6:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP71:%.*]] = load i32, ptr [[ARRAYIDX13_3_6]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_6:%.*]] = add nsw i32 [[TMP71]], [[TMP70]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_6]], ptr [[ARRAYIDX13_3_6]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_6:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.6: +; CHECK-UNROLL-NEXT: [[CMP1_7:%.*]] = icmp eq i32 7, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_7]], label [[CLEANUP]], label [[IF_END_7:%.*]] +; CHECK-UNROLL: if.end.7: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 7 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_7:%.*]] +; CHECK-UNROLL: for.cond4.7: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_7:%.*]] +; CHECK-UNROLL: for.body7.7: +; CHECK-UNROLL-NEXT: [[TMP72:%.*]] = load ptr, ptr [[ARRAYIDX_7]], align 8 +; CHECK-UNROLL-NEXT: [[TMP73:%.*]] = load i32, ptr [[TMP72]], align 4 +; CHECK-UNROLL-NEXT: [[TMP74:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_7:%.*]] = add nsw i32 [[TMP74]], [[TMP73]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_7]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_7:%.*]] +; CHECK-UNROLL: for.body7.1.7: +; CHECK-UNROLL-NEXT: [[TMP75:%.*]] = load ptr, ptr [[ARRAYIDX_7]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_7:%.*]] = getelementptr inbounds i32, ptr [[TMP75]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP76:%.*]] = load i32, ptr [[ARRAYIDX11_1_7]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_7:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP77:%.*]] = load i32, ptr [[ARRAYIDX13_1_7]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_7:%.*]] = add nsw i32 [[TMP77]], [[TMP76]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_7]], ptr [[ARRAYIDX13_1_7]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_7:%.*]] +; CHECK-UNROLL: for.body7.2.7: +; CHECK-UNROLL-NEXT: [[TMP78:%.*]] = load ptr, ptr [[ARRAYIDX_7]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_7:%.*]] = getelementptr inbounds i32, ptr [[TMP78]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP79:%.*]] = load i32, ptr [[ARRAYIDX11_2_7]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_7:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP80:%.*]] = load i32, ptr [[ARRAYIDX13_2_7]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_7:%.*]] = add nsw i32 [[TMP80]], [[TMP79]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_7]], ptr [[ARRAYIDX13_2_7]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_7:%.*]] +; CHECK-UNROLL: for.body7.3.7: +; CHECK-UNROLL-NEXT: [[TMP81:%.*]] = load ptr, ptr [[ARRAYIDX_7]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_7:%.*]] = getelementptr inbounds i32, ptr [[TMP81]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP82:%.*]] = load i32, ptr [[ARRAYIDX11_3_7]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_7:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP83:%.*]] = load i32, ptr [[ARRAYIDX13_3_7]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_7:%.*]] = add nsw i32 [[TMP83]], [[TMP82]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_7]], ptr [[ARRAYIDX13_3_7]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_7:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.7: +; CHECK-UNROLL-NEXT: [[CMP1_8:%.*]] = icmp eq i32 8, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_8]], label [[CLEANUP]], label [[IF_END_8:%.*]] +; CHECK-UNROLL: if.end.8: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_8:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 8 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_8:%.*]] +; CHECK-UNROLL: for.cond4.8: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_8:%.*]] +; CHECK-UNROLL: for.body7.8: +; CHECK-UNROLL-NEXT: [[TMP84:%.*]] = load ptr, ptr [[ARRAYIDX_8]], align 8 +; CHECK-UNROLL-NEXT: [[TMP85:%.*]] = load i32, ptr [[TMP84]], align 4 +; CHECK-UNROLL-NEXT: [[TMP86:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_8:%.*]] = add nsw i32 [[TMP86]], [[TMP85]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_8]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_8:%.*]] +; CHECK-UNROLL: for.body7.1.8: +; CHECK-UNROLL-NEXT: [[TMP87:%.*]] = load ptr, ptr [[ARRAYIDX_8]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_8:%.*]] = getelementptr inbounds i32, ptr [[TMP87]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP88:%.*]] = load i32, ptr [[ARRAYIDX11_1_8]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_8:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP89:%.*]] = load i32, ptr [[ARRAYIDX13_1_8]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_8:%.*]] = add nsw i32 [[TMP89]], [[TMP88]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_8]], ptr [[ARRAYIDX13_1_8]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_8:%.*]] +; CHECK-UNROLL: for.body7.2.8: +; CHECK-UNROLL-NEXT: [[TMP90:%.*]] = load ptr, ptr [[ARRAYIDX_8]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_8:%.*]] = getelementptr inbounds i32, ptr [[TMP90]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP91:%.*]] = load i32, ptr [[ARRAYIDX11_2_8]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_8:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP92:%.*]] = load i32, ptr [[ARRAYIDX13_2_8]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_8:%.*]] = add nsw i32 [[TMP92]], [[TMP91]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_8]], ptr [[ARRAYIDX13_2_8]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_8:%.*]] +; CHECK-UNROLL: for.body7.3.8: +; CHECK-UNROLL-NEXT: [[TMP93:%.*]] = load ptr, ptr [[ARRAYIDX_8]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_8:%.*]] = getelementptr inbounds i32, ptr [[TMP93]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP94:%.*]] = load i32, ptr [[ARRAYIDX11_3_8]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_8:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP95:%.*]] = load i32, ptr [[ARRAYIDX13_3_8]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_8:%.*]] = add nsw i32 [[TMP95]], [[TMP94]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_8]], ptr [[ARRAYIDX13_3_8]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_8:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.8: +; CHECK-UNROLL-NEXT: [[CMP1_9:%.*]] = icmp eq i32 9, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_9]], label [[CLEANUP]], label [[IF_END_9:%.*]] +; CHECK-UNROLL: if.end.9: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_9:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 9 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_9:%.*]] +; CHECK-UNROLL: for.cond4.9: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_9:%.*]] +; CHECK-UNROLL: for.body7.9: +; CHECK-UNROLL-NEXT: [[TMP96:%.*]] = load ptr, ptr [[ARRAYIDX_9]], align 8 +; CHECK-UNROLL-NEXT: [[TMP97:%.*]] = load i32, ptr [[TMP96]], align 4 +; CHECK-UNROLL-NEXT: [[TMP98:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_9:%.*]] = add nsw i32 [[TMP98]], [[TMP97]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_9]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_9:%.*]] +; CHECK-UNROLL: for.body7.1.9: +; CHECK-UNROLL-NEXT: [[TMP99:%.*]] = load ptr, ptr [[ARRAYIDX_9]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_9:%.*]] = getelementptr inbounds i32, ptr [[TMP99]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP100:%.*]] = load i32, ptr [[ARRAYIDX11_1_9]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_9:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP101:%.*]] = load i32, ptr [[ARRAYIDX13_1_9]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_9:%.*]] = add nsw i32 [[TMP101]], [[TMP100]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_9]], ptr [[ARRAYIDX13_1_9]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_9:%.*]] +; CHECK-UNROLL: for.body7.2.9: +; CHECK-UNROLL-NEXT: [[TMP102:%.*]] = load ptr, ptr [[ARRAYIDX_9]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_9:%.*]] = getelementptr inbounds i32, ptr [[TMP102]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP103:%.*]] = load i32, ptr [[ARRAYIDX11_2_9]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_9:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP104:%.*]] = load i32, ptr [[ARRAYIDX13_2_9]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_9:%.*]] = add nsw i32 [[TMP104]], [[TMP103]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_9]], ptr [[ARRAYIDX13_2_9]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_9:%.*]] +; CHECK-UNROLL: for.body7.3.9: +; CHECK-UNROLL-NEXT: [[TMP105:%.*]] = load ptr, ptr [[ARRAYIDX_9]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_9:%.*]] = getelementptr inbounds i32, ptr [[TMP105]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP106:%.*]] = load i32, ptr [[ARRAYIDX11_3_9]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_9:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP107:%.*]] = load i32, ptr [[ARRAYIDX13_3_9]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_9:%.*]] = add nsw i32 [[TMP107]], [[TMP106]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_9]], ptr [[ARRAYIDX13_3_9]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_9:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.9: +; CHECK-UNROLL-NEXT: [[CMP1_10:%.*]] = icmp eq i32 10, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_10]], label [[CLEANUP]], label [[IF_END_10:%.*]] +; CHECK-UNROLL: if.end.10: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_10:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 10 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_10:%.*]] +; CHECK-UNROLL: for.cond4.10: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_10:%.*]] +; CHECK-UNROLL: for.body7.10: +; CHECK-UNROLL-NEXT: [[TMP108:%.*]] = load ptr, ptr [[ARRAYIDX_10]], align 8 +; CHECK-UNROLL-NEXT: [[TMP109:%.*]] = load i32, ptr [[TMP108]], align 4 +; CHECK-UNROLL-NEXT: [[TMP110:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_10:%.*]] = add nsw i32 [[TMP110]], [[TMP109]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_10]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_10:%.*]] +; CHECK-UNROLL: for.body7.1.10: +; CHECK-UNROLL-NEXT: [[TMP111:%.*]] = load ptr, ptr [[ARRAYIDX_10]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_10:%.*]] = getelementptr inbounds i32, ptr [[TMP111]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP112:%.*]] = load i32, ptr [[ARRAYIDX11_1_10]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_10:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP113:%.*]] = load i32, ptr [[ARRAYIDX13_1_10]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_10:%.*]] = add nsw i32 [[TMP113]], [[TMP112]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_10]], ptr [[ARRAYIDX13_1_10]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_10:%.*]] +; CHECK-UNROLL: for.body7.2.10: +; CHECK-UNROLL-NEXT: [[TMP114:%.*]] = load ptr, ptr [[ARRAYIDX_10]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_10:%.*]] = getelementptr inbounds i32, ptr [[TMP114]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP115:%.*]] = load i32, ptr [[ARRAYIDX11_2_10]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_10:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP116:%.*]] = load i32, ptr [[ARRAYIDX13_2_10]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_10:%.*]] = add nsw i32 [[TMP116]], [[TMP115]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_10]], ptr [[ARRAYIDX13_2_10]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_10:%.*]] +; CHECK-UNROLL: for.body7.3.10: +; CHECK-UNROLL-NEXT: [[TMP117:%.*]] = load ptr, ptr [[ARRAYIDX_10]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_10:%.*]] = getelementptr inbounds i32, ptr [[TMP117]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP118:%.*]] = load i32, ptr [[ARRAYIDX11_3_10]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_10:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP119:%.*]] = load i32, ptr [[ARRAYIDX13_3_10]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_10:%.*]] = add nsw i32 [[TMP119]], [[TMP118]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_10]], ptr [[ARRAYIDX13_3_10]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_10:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.10: +; CHECK-UNROLL-NEXT: [[CMP1_11:%.*]] = icmp eq i32 11, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_11]], label [[CLEANUP]], label [[IF_END_11:%.*]] +; CHECK-UNROLL: if.end.11: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_11:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 11 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_11:%.*]] +; CHECK-UNROLL: for.cond4.11: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_11:%.*]] +; CHECK-UNROLL: for.body7.11: +; CHECK-UNROLL-NEXT: [[TMP120:%.*]] = load ptr, ptr [[ARRAYIDX_11]], align 8 +; CHECK-UNROLL-NEXT: [[TMP121:%.*]] = load i32, ptr [[TMP120]], align 4 +; CHECK-UNROLL-NEXT: [[TMP122:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_119:%.*]] = add nsw i32 [[TMP122]], [[TMP121]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_119]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_11:%.*]] +; CHECK-UNROLL: for.body7.1.11: +; CHECK-UNROLL-NEXT: [[TMP123:%.*]] = load ptr, ptr [[ARRAYIDX_11]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_11:%.*]] = getelementptr inbounds i32, ptr [[TMP123]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP124:%.*]] = load i32, ptr [[ARRAYIDX11_1_11]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_11:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP125:%.*]] = load i32, ptr [[ARRAYIDX13_1_11]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_11:%.*]] = add nsw i32 [[TMP125]], [[TMP124]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_11]], ptr [[ARRAYIDX13_1_11]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_11:%.*]] +; CHECK-UNROLL: for.body7.2.11: +; CHECK-UNROLL-NEXT: [[TMP126:%.*]] = load ptr, ptr [[ARRAYIDX_11]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_11:%.*]] = getelementptr inbounds i32, ptr [[TMP126]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP127:%.*]] = load i32, ptr [[ARRAYIDX11_2_11]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_11:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP128:%.*]] = load i32, ptr [[ARRAYIDX13_2_11]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_11:%.*]] = add nsw i32 [[TMP128]], [[TMP127]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_11]], ptr [[ARRAYIDX13_2_11]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_11:%.*]] +; CHECK-UNROLL: for.body7.3.11: +; CHECK-UNROLL-NEXT: [[TMP129:%.*]] = load ptr, ptr [[ARRAYIDX_11]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_11:%.*]] = getelementptr inbounds i32, ptr [[TMP129]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP130:%.*]] = load i32, ptr [[ARRAYIDX11_3_11]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_11:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP131:%.*]] = load i32, ptr [[ARRAYIDX13_3_11]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_11:%.*]] = add nsw i32 [[TMP131]], [[TMP130]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_11]], ptr [[ARRAYIDX13_3_11]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_11:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.11: +; CHECK-UNROLL-NEXT: [[CMP1_12:%.*]] = icmp eq i32 12, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_12]], label [[CLEANUP]], label [[IF_END_12:%.*]] +; CHECK-UNROLL: if.end.12: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_12:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 12 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_12:%.*]] +; CHECK-UNROLL: for.cond4.12: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1210:%.*]] +; CHECK-UNROLL: for.body7.1210: +; CHECK-UNROLL-NEXT: [[TMP132:%.*]] = load ptr, ptr [[ARRAYIDX_12]], align 8 +; CHECK-UNROLL-NEXT: [[TMP133:%.*]] = load i32, ptr [[TMP132]], align 4 +; CHECK-UNROLL-NEXT: [[TMP134:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_12:%.*]] = add nsw i32 [[TMP134]], [[TMP133]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_12]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_12:%.*]] +; CHECK-UNROLL: for.body7.1.12: +; CHECK-UNROLL-NEXT: [[TMP135:%.*]] = load ptr, ptr [[ARRAYIDX_12]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_12:%.*]] = getelementptr inbounds i32, ptr [[TMP135]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP136:%.*]] = load i32, ptr [[ARRAYIDX11_1_12]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_12:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP137:%.*]] = load i32, ptr [[ARRAYIDX13_1_12]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_12:%.*]] = add nsw i32 [[TMP137]], [[TMP136]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_12]], ptr [[ARRAYIDX13_1_12]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_12:%.*]] +; CHECK-UNROLL: for.body7.2.12: +; CHECK-UNROLL-NEXT: [[TMP138:%.*]] = load ptr, ptr [[ARRAYIDX_12]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_12:%.*]] = getelementptr inbounds i32, ptr [[TMP138]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP139:%.*]] = load i32, ptr [[ARRAYIDX11_2_12]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_12:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP140:%.*]] = load i32, ptr [[ARRAYIDX13_2_12]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_12:%.*]] = add nsw i32 [[TMP140]], [[TMP139]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_12]], ptr [[ARRAYIDX13_2_12]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_12:%.*]] +; CHECK-UNROLL: for.body7.3.12: +; CHECK-UNROLL-NEXT: [[TMP141:%.*]] = load ptr, ptr [[ARRAYIDX_12]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_12:%.*]] = getelementptr inbounds i32, ptr [[TMP141]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP142:%.*]] = load i32, ptr [[ARRAYIDX11_3_12]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_12:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP143:%.*]] = load i32, ptr [[ARRAYIDX13_3_12]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_12:%.*]] = add nsw i32 [[TMP143]], [[TMP142]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_12]], ptr [[ARRAYIDX13_3_12]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_12:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.12: +; CHECK-UNROLL-NEXT: [[CMP1_13:%.*]] = icmp eq i32 13, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_13]], label [[CLEANUP]], label [[IF_END_13:%.*]] +; CHECK-UNROLL: if.end.13: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_13:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 13 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_13:%.*]] +; CHECK-UNROLL: for.cond4.13: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_13:%.*]] +; CHECK-UNROLL: for.body7.13: +; CHECK-UNROLL-NEXT: [[TMP144:%.*]] = load ptr, ptr [[ARRAYIDX_13]], align 8 +; CHECK-UNROLL-NEXT: [[TMP145:%.*]] = load i32, ptr [[TMP144]], align 4 +; CHECK-UNROLL-NEXT: [[TMP146:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_13:%.*]] = add nsw i32 [[TMP146]], [[TMP145]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_13]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_13:%.*]] +; CHECK-UNROLL: for.body7.1.13: +; CHECK-UNROLL-NEXT: [[TMP147:%.*]] = load ptr, ptr [[ARRAYIDX_13]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_13:%.*]] = getelementptr inbounds i32, ptr [[TMP147]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP148:%.*]] = load i32, ptr [[ARRAYIDX11_1_13]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_13:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP149:%.*]] = load i32, ptr [[ARRAYIDX13_1_13]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_13:%.*]] = add nsw i32 [[TMP149]], [[TMP148]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_13]], ptr [[ARRAYIDX13_1_13]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_13:%.*]] +; CHECK-UNROLL: for.body7.2.13: +; CHECK-UNROLL-NEXT: [[TMP150:%.*]] = load ptr, ptr [[ARRAYIDX_13]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_13:%.*]] = getelementptr inbounds i32, ptr [[TMP150]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP151:%.*]] = load i32, ptr [[ARRAYIDX11_2_13]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_13:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP152:%.*]] = load i32, ptr [[ARRAYIDX13_2_13]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_13:%.*]] = add nsw i32 [[TMP152]], [[TMP151]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_13]], ptr [[ARRAYIDX13_2_13]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_13:%.*]] +; CHECK-UNROLL: for.body7.3.13: +; CHECK-UNROLL-NEXT: [[TMP153:%.*]] = load ptr, ptr [[ARRAYIDX_13]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_13:%.*]] = getelementptr inbounds i32, ptr [[TMP153]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP154:%.*]] = load i32, ptr [[ARRAYIDX11_3_13]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_13:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP155:%.*]] = load i32, ptr [[ARRAYIDX13_3_13]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_13:%.*]] = add nsw i32 [[TMP155]], [[TMP154]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_13]], ptr [[ARRAYIDX13_3_13]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_13:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.13: +; CHECK-UNROLL-NEXT: [[CMP1_14:%.*]] = icmp eq i32 14, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_14]], label [[CLEANUP]], label [[IF_END_14:%.*]] +; CHECK-UNROLL: if.end.14: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_14:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 14 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_14:%.*]] +; CHECK-UNROLL: for.cond4.14: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_14:%.*]] +; CHECK-UNROLL: for.body7.14: +; CHECK-UNROLL-NEXT: [[TMP156:%.*]] = load ptr, ptr [[ARRAYIDX_14]], align 8 +; CHECK-UNROLL-NEXT: [[TMP157:%.*]] = load i32, ptr [[TMP156]], align 4 +; CHECK-UNROLL-NEXT: [[TMP158:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_14:%.*]] = add nsw i32 [[TMP158]], [[TMP157]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_14]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_14:%.*]] +; CHECK-UNROLL: for.body7.1.14: +; CHECK-UNROLL-NEXT: [[TMP159:%.*]] = load ptr, ptr [[ARRAYIDX_14]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_14:%.*]] = getelementptr inbounds i32, ptr [[TMP159]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP160:%.*]] = load i32, ptr [[ARRAYIDX11_1_14]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_14:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP161:%.*]] = load i32, ptr [[ARRAYIDX13_1_14]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_14:%.*]] = add nsw i32 [[TMP161]], [[TMP160]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_14]], ptr [[ARRAYIDX13_1_14]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_14:%.*]] +; CHECK-UNROLL: for.body7.2.14: +; CHECK-UNROLL-NEXT: [[TMP162:%.*]] = load ptr, ptr [[ARRAYIDX_14]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_14:%.*]] = getelementptr inbounds i32, ptr [[TMP162]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP163:%.*]] = load i32, ptr [[ARRAYIDX11_2_14]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_14:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP164:%.*]] = load i32, ptr [[ARRAYIDX13_2_14]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_14:%.*]] = add nsw i32 [[TMP164]], [[TMP163]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_14]], ptr [[ARRAYIDX13_2_14]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_14:%.*]] +; CHECK-UNROLL: for.body7.3.14: +; CHECK-UNROLL-NEXT: [[TMP165:%.*]] = load ptr, ptr [[ARRAYIDX_14]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_14:%.*]] = getelementptr inbounds i32, ptr [[TMP165]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP166:%.*]] = load i32, ptr [[ARRAYIDX11_3_14]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_14:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP167:%.*]] = load i32, ptr [[ARRAYIDX13_3_14]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_14:%.*]] = add nsw i32 [[TMP167]], [[TMP166]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_14]], ptr [[ARRAYIDX13_3_14]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_14:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.14: +; CHECK-UNROLL-NEXT: [[CMP1_15:%.*]] = icmp eq i32 15, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_15]], label [[CLEANUP]], label [[IF_END_15:%.*]] +; CHECK-UNROLL: if.end.15: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_15:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 15 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_15:%.*]] +; CHECK-UNROLL: for.cond4.15: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_15:%.*]] +; CHECK-UNROLL: for.body7.15: +; CHECK-UNROLL-NEXT: [[TMP168:%.*]] = load ptr, ptr [[ARRAYIDX_15]], align 8 +; CHECK-UNROLL-NEXT: [[TMP169:%.*]] = load i32, ptr [[TMP168]], align 4 +; CHECK-UNROLL-NEXT: [[TMP170:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_15:%.*]] = add nsw i32 [[TMP170]], [[TMP169]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_15]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_15:%.*]] +; CHECK-UNROLL: for.body7.1.15: +; CHECK-UNROLL-NEXT: [[TMP171:%.*]] = load ptr, ptr [[ARRAYIDX_15]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_15:%.*]] = getelementptr inbounds i32, ptr [[TMP171]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP172:%.*]] = load i32, ptr [[ARRAYIDX11_1_15]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_15:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP173:%.*]] = load i32, ptr [[ARRAYIDX13_1_15]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_15:%.*]] = add nsw i32 [[TMP173]], [[TMP172]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_15]], ptr [[ARRAYIDX13_1_15]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_15:%.*]] +; CHECK-UNROLL: for.body7.2.15: +; CHECK-UNROLL-NEXT: [[TMP174:%.*]] = load ptr, ptr [[ARRAYIDX_15]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_15:%.*]] = getelementptr inbounds i32, ptr [[TMP174]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP175:%.*]] = load i32, ptr [[ARRAYIDX11_2_15]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_15:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP176:%.*]] = load i32, ptr [[ARRAYIDX13_2_15]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_15:%.*]] = add nsw i32 [[TMP176]], [[TMP175]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_15]], ptr [[ARRAYIDX13_2_15]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_15:%.*]] +; CHECK-UNROLL: for.body7.3.15: +; CHECK-UNROLL-NEXT: [[TMP177:%.*]] = load ptr, ptr [[ARRAYIDX_15]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_15:%.*]] = getelementptr inbounds i32, ptr [[TMP177]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP178:%.*]] = load i32, ptr [[ARRAYIDX11_3_15]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_15:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP179:%.*]] = load i32, ptr [[ARRAYIDX13_3_15]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_15:%.*]] = add nsw i32 [[TMP179]], [[TMP178]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_15]], ptr [[ARRAYIDX13_3_15]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_15:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.15: +; CHECK-UNROLL-NEXT: br i1 true, label [[CLEANUP]], label [[IF_END_16:%.*]] +; CHECK-UNROLL: if.end.16: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_16:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 16 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_16:%.*]] +; CHECK-UNROLL: for.cond4.16: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_16:%.*]] +; CHECK-UNROLL: for.body7.16: +; CHECK-UNROLL-NEXT: [[TMP180:%.*]] = load ptr, ptr [[ARRAYIDX_16]], align 8 +; CHECK-UNROLL-NEXT: [[TMP181:%.*]] = load i32, ptr [[TMP180]], align 4 +; CHECK-UNROLL-NEXT: [[TMP182:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_16:%.*]] = add nsw i32 [[TMP182]], [[TMP181]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_16]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_16:%.*]] +; CHECK-UNROLL: for.body7.1.16: +; CHECK-UNROLL-NEXT: [[TMP183:%.*]] = load ptr, ptr [[ARRAYIDX_16]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_16:%.*]] = getelementptr inbounds i32, ptr [[TMP183]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP184:%.*]] = load i32, ptr [[ARRAYIDX11_1_16]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_16:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP185:%.*]] = load i32, ptr [[ARRAYIDX13_1_16]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_16:%.*]] = add nsw i32 [[TMP185]], [[TMP184]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_16]], ptr [[ARRAYIDX13_1_16]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_16:%.*]] +; CHECK-UNROLL: for.body7.2.16: +; CHECK-UNROLL-NEXT: [[TMP186:%.*]] = load ptr, ptr [[ARRAYIDX_16]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_16:%.*]] = getelementptr inbounds i32, ptr [[TMP186]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP187:%.*]] = load i32, ptr [[ARRAYIDX11_2_16]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_16:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP188:%.*]] = load i32, ptr [[ARRAYIDX13_2_16]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_16:%.*]] = add nsw i32 [[TMP188]], [[TMP187]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_16]], ptr [[ARRAYIDX13_2_16]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_16:%.*]] +; CHECK-UNROLL: for.body7.3.16: +; CHECK-UNROLL-NEXT: [[TMP189:%.*]] = load ptr, ptr [[ARRAYIDX_16]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_16:%.*]] = getelementptr inbounds i32, ptr [[TMP189]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP190:%.*]] = load i32, ptr [[ARRAYIDX11_3_16]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_16:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP191:%.*]] = load i32, ptr [[ARRAYIDX13_3_16]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_16:%.*]] = add nsw i32 [[TMP191]], [[TMP190]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_16]], ptr [[ARRAYIDX13_3_16]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_16:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.16: +; CHECK-UNROLL-NEXT: unreachable +; CHECK-UNROLL: for.body7: +; CHECK-UNROLL-NEXT: [[TMP192:%.*]] = load ptr, ptr [[ARR]], align 8 +; CHECK-UNROLL-NEXT: [[TMP193:%.*]] = load i32, ptr [[TMP192]], align 4 +; CHECK-UNROLL-NEXT: [[TMP194:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14:%.*]] = add nsw i32 [[TMP194]], [[TMP193]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1:%.*]] +; CHECK-UNROLL: for.body7.1: +; CHECK-UNROLL-NEXT: [[TMP195:%.*]] = load ptr, ptr [[ARR]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1:%.*]] = getelementptr inbounds i32, ptr [[TMP195]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP196:%.*]] = load i32, ptr [[ARRAYIDX11_1]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP197:%.*]] = load i32, ptr [[ARRAYIDX13_1]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1:%.*]] = add nsw i32 [[TMP197]], [[TMP196]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1]], ptr [[ARRAYIDX13_1]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2:%.*]] +; CHECK-UNROLL: for.body7.2: +; CHECK-UNROLL-NEXT: [[TMP198:%.*]] = load ptr, ptr [[ARR]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2:%.*]] = getelementptr inbounds i32, ptr [[TMP198]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP199:%.*]] = load i32, ptr [[ARRAYIDX11_2]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP200:%.*]] = load i32, ptr [[ARRAYIDX13_2]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2:%.*]] = add nsw i32 [[TMP200]], [[TMP199]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2]], ptr [[ARRAYIDX13_2]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3:%.*]] +; CHECK-UNROLL: for.body7.3: +; CHECK-UNROLL-NEXT: [[TMP201:%.*]] = load ptr, ptr [[ARR]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3:%.*]] = getelementptr inbounds i32, ptr [[TMP201]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP202:%.*]] = load i32, ptr [[ARRAYIDX11_3]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP203:%.*]] = load i32, ptr [[ARRAYIDX13_3]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3:%.*]] = add nsw i32 [[TMP203]], [[TMP202]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3]], ptr [[ARRAYIDX13_3]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6:%.*]] +; CHECK-UNROLL: for.body7.4: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_LCSSA:%.*]] = phi ptr [ [[ARR]], [[FOR_BODY7_3]] ], [ [[ARRAYIDX_1]], [[FOR_BODY7_3_1]] ], [ [[ARRAYIDX_2]], [[FOR_BODY7_3_2]] ], [ [[ARRAYIDX_3]], [[FOR_BODY7_3_3]] ], [ [[ARRAYIDX_4]], [[FOR_BODY7_3_4]] ], [ [[ARRAYIDX_5]], [[FOR_BODY7_3_5]] ], [ [[ARRAYIDX_6]], [[FOR_BODY7_3_6]] ], [ [[ARRAYIDX_7]], [[FOR_BODY7_3_7]] ], [ [[ARRAYIDX_8]], [[FOR_BODY7_3_8]] ], [ [[ARRAYIDX_9]], [[FOR_BODY7_3_9]] ], [ [[ARRAYIDX_10]], [[FOR_BODY7_3_10]] ], [ [[ARRAYIDX_11]], [[FOR_BODY7_3_11]] ], [ [[ARRAYIDX_12]], [[FOR_BODY7_3_12]] ], [ [[ARRAYIDX_13]], [[FOR_BODY7_3_13]] ], [ [[ARRAYIDX_14]], [[FOR_BODY7_3_14]] ], [ [[ARRAYIDX_15]], [[FOR_BODY7_3_15]] ], [ [[ARRAYIDX_16]], [[FOR_BODY7_3_16]] ] +; CHECK-UNROLL-NEXT: [[TMP204:%.*]] = load ptr, ptr [[ARRAYIDX_LCSSA]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_4:%.*]] = getelementptr inbounds i32, ptr [[TMP204]], i64 4 +; CHECK-UNROLL-NEXT: [[TMP205:%.*]] = load i32, ptr [[ARRAYIDX11_4]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_4:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 4 +; CHECK-UNROLL-NEXT: [[TMP206:%.*]] = load i32, ptr [[ARRAYIDX13_4]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_4:%.*]] = add nsw i32 [[TMP206]], [[TMP205]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_4]], ptr [[ARRAYIDX13_4]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: unreachable +; CHECK-UNROLL: cleanup: +; CHECK-UNROLL-NEXT: ret void +; +entry: + br label %for.cond + +for.cond: ; preds = %for.cond.cleanup6, %entry + %Dim.0 = phi i32 [ 0, %entry ], [ %inc16, %for.cond.cleanup6 ] + %Idx.addr.0 = phi i32 [ %Idx, %entry ], [ %add, %for.cond.cleanup6 ] + %cmp = icmp slt i32 %Dim.0, 16 + br i1 %cmp, label %for.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %for.cond + br label %cleanup + +for.body: ; preds = %for.cond + %cmp1 = icmp eq i32 %Dim.0, %Dims + br i1 %cmp1, label %if.then, label %if.end + +if.then: ; preds = %for.body + br label %cleanup + +if.end: ; preds = %for.body + %idxprom = sext i32 %Dim.0 to i64 + %arrayidx = getelementptr inbounds ptr, ptr %Arr, i64 %idxprom + %0 = load ptr, ptr %arrayidx, align 8 + %idxprom2 = sext i32 %Idx.addr.0 to i64 + %arrayidx3 = getelementptr inbounds i32, ptr %0, i64 %idxprom2 + %1 = load i32, ptr %arrayidx3, align 4 + %add = add nsw i32 %1, 1 + br label %for.cond4 + +for.cond4: ; preds = %for.body7, %if.end + %arg.0 = phi i32 [ 0, %if.end ], [ %inc, %for.body7 ] + %cmp5 = icmp slt i32 %arg.0, 4 + br i1 %cmp5, label %for.body7, label %for.cond.cleanup6 + +for.cond.cleanup6: ; preds = %for.cond4 + %inc16 = add nsw i32 %Dim.0, 1 + br label %for.cond, !llvm.loop !0 + +for.body7: ; preds = %for.cond4 + %2 = load ptr, ptr %arrayidx, align 8 + %idxprom10 = sext i32 %arg.0 to i64 + %arrayidx11 = getelementptr inbounds i32, ptr %2, i64 %idxprom10 + %3 = load i32, ptr %arrayidx11, align 4 + %arrayidx13 = getelementptr inbounds i32, ptr %Out, i64 %idxprom10 + %4 = load i32, ptr %arrayidx13, align 4 + %add14 = add nsw i32 %4, %3 + store i32 %add14, ptr %arrayidx13, align 4 + call void @_Z3barv() + %inc = add nsw i32 %arg.0, 1 + br label %for.cond4, !llvm.loop !3 + +cleanup: ; preds = %if.then, %for.cond.cleanup + ret void +} + + declare void @_Z3barv() + +!0 = distinct !{!0, !1, !2} +!1 = !{!"llvm.loop.mustprogress"} +!2 = !{!"llvm.loop.unroll.enable"} +!3 = distinct !{!3, !1} +;. +; CHECK-CFG: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} +; CHECK-CFG: [[META1]] = !{!"llvm.loop.mustprogress"} +; CHECK-CFG: [[META2]] = !{!"llvm.loop.unroll.enable"} +; CHECK-CFG: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]} +;. diff --git a/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll b/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll index cf875ccdc147ab96d2f175b79c9a01ca3fbcc8b8..669306c8f3ab747873ccba4f21ffbcc17a0413f7 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll @@ -96,7 +96,7 @@ define void @pr56282() { ; CHECK: inner.2.preheader: ; CHECK-NEXT: br label [[INNER_2]] ; CHECK: inner.2: -; CHECK-NEXT: [[OUTER_IV_NEXT]] = add i64 [[OUTER_IV]], 1 +; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw i64 [[OUTER_IV]], 1 ; CHECK-NEXT: br label [[OUTER_HEADER]] ; CHECK: exit: ; CHECK-NEXT: ret void diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll index 6b5d69d100dde85d7f328e60dec6d6dd79d9b701..8c50d86489c9ddd4051305eb9932e1e6f4bbc02e 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll @@ -18,16 +18,16 @@ define void @f1(ptr %A) #0 { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0 -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer), ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0 +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer), ptr [[TMP8]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll b/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll index 98d55ae15c07761cfa78fb608d7e5ddaedc05387..763b3e0bc8293012948b8e306dceb47b2e7233ef 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll @@ -54,6 +54,8 @@ define dso_local double @test(ptr nocapture noundef readonly %data, ptr nocaptur ; SVE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SVE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]] ; SVE-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]] +; SVE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; SVE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 ; SVE-NEXT: br label [[VECTOR_BODY:%.*]] ; SVE: vector.body: ; SVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -66,8 +68,6 @@ define dso_local double @test(ptr nocapture noundef readonly %data, ptr nocaptur ; SVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[DATA:%.*]], [[TMP7]] ; SVE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv2f64.nxv2p0( [[TMP8]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) ; SVE-NEXT: [[TMP9]] = fadd [[VEC_PHI]], [[WIDE_MASKED_GATHER]] -; SVE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; SVE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 ; SVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; SVE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SVE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll index 28962dfba89248a02b815476dc6d2d8541ff624b..144b29d84198ac8780553483c67412e94549dcd2 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll @@ -19,6 +19,8 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; TFNONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; TFNONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]] ; TFNONE: vector.body: ; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -27,8 +29,6 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP5:%.*]] = call @foo_vector( [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; TFNONE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFNONE-NEXT: store [[TMP5]], ptr [[TMP6]], align 8 -; TFNONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; TFNONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; TFNONE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TFNONE-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -60,6 +60,8 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 { ; TFCOMMON-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TFCOMMON-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TFCOMMON-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TFCOMMON-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; TFCOMMON-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; TFCOMMON-NEXT: br label [[VECTOR_BODY:%.*]] ; TFCOMMON: vector.body: @@ -70,8 +72,6 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 { ; TFCOMMON-NEXT: [[TMP6:%.*]] = call @foo_vector( [[WIDE_MASKED_LOAD]], [[ACTIVE_LANE_MASK]]) ; TFCOMMON-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFCOMMON-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP6]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; TFCOMMON-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; TFCOMMON-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TFCOMMON-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] ; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; TFCOMMON-NEXT: [[TMP10:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -111,6 +111,8 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; TFNONE-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; TFNONE-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]] ; TFNONE: vector.body: ; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -122,8 +124,6 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[PREDPHI:%.*]] = select [[TMP7]], zeroinitializer, [[TMP6]] ; TFNONE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] ; TFNONE-NEXT: store [[PREDPHI]], ptr [[TMP8]], align 8 -; TFNONE-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; TFNONE-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] ; TFNONE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TFNONE-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -161,6 +161,8 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 { ; TFCOMMON-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TFCOMMON-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TFCOMMON-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TFCOMMON-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() +; TFCOMMON-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 2 ; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; TFCOMMON-NEXT: br label [[VECTOR_BODY:%.*]] ; TFCOMMON: vector.body: @@ -177,8 +179,6 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 { ; TFCOMMON-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] ; TFCOMMON-NEXT: [[TMP12:%.*]] = or [[TMP7]], [[TMP10]] ; TFCOMMON-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[PREDPHI]], ptr [[TMP11]], i32 8, [[TMP12]]) -; TFCOMMON-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; TFCOMMON-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 2 ; TFCOMMON-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP14]] ; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; TFCOMMON-NEXT: [[TMP15:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -229,6 +229,8 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; TFNONE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; TFNONE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 ; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]] ; TFNONE: vector.body: ; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -241,8 +243,6 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[PREDPHI:%.*]] = select [[TMP6]], [[TMP7]], [[TMP8]] ; TFNONE-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] ; TFNONE-NEXT: store [[PREDPHI]], ptr [[TMP9]], align 8 -; TFNONE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; TFNONE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 ; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; TFNONE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TFNONE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -283,6 +283,8 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 { ; TFCOMMON-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TFCOMMON-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TFCOMMON-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TFCOMMON-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; TFCOMMON-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 2 ; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; TFCOMMON-NEXT: br label [[VECTOR_BODY:%.*]] ; TFCOMMON: vector.body: @@ -300,8 +302,6 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 { ; TFCOMMON-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] ; TFCOMMON-NEXT: [[TMP13:%.*]] = or [[TMP8]], [[TMP10]] ; TFCOMMON-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[PREDPHI]], ptr [[TMP12]], i32 8, [[TMP13]]) -; TFCOMMON-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; TFCOMMON-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 2 ; TFCOMMON-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP15]] ; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; TFCOMMON-NEXT: [[TMP16:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -355,6 +355,8 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; TFNONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; TFNONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]] ; TFNONE: vector.body: ; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -363,8 +365,6 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP5:%.*]] = call @foo_vector_nomask( [[WIDE_LOAD]]) ; TFNONE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFNONE-NEXT: store [[TMP5]], ptr [[TMP6]], align 8 -; TFNONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; TFNONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; TFNONE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TFNONE-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -413,6 +413,8 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 { ; TFFALLBACK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; TFFALLBACK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; TFFALLBACK-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; TFFALLBACK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; TFFALLBACK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFFALLBACK-NEXT: br label [[VECTOR_BODY:%.*]] ; TFFALLBACK: vector.body: ; TFFALLBACK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -421,8 +423,6 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 { ; TFFALLBACK-NEXT: [[TMP5:%.*]] = call @foo_vector_nomask( [[WIDE_LOAD]]) ; TFFALLBACK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFFALLBACK-NEXT: store [[TMP5]], ptr [[TMP6]], align 8 -; TFFALLBACK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; TFFALLBACK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFFALLBACK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; TFFALLBACK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TFFALLBACK-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] @@ -475,6 +475,8 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; TFNONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; TFNONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]] ; TFNONE: vector.body: ; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -483,8 +485,6 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP5:%.*]] = call @foo_vector_nomask( [[WIDE_LOAD]]) ; TFNONE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFNONE-NEXT: store [[TMP5]], ptr [[TMP6]], align 8 -; TFNONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; TFNONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; TFNONE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TFNONE-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -516,6 +516,8 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 { ; TFALWAYS-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TFALWAYS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TFALWAYS-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TFALWAYS-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; TFALWAYS-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; TFALWAYS-NEXT: br label [[VECTOR_BODY:%.*]] ; TFALWAYS: vector.body: @@ -526,8 +528,6 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 { ; TFALWAYS-NEXT: [[TMP6:%.*]] = call @foo_vector( [[WIDE_MASKED_LOAD]], [[ACTIVE_LANE_MASK]]) ; TFALWAYS-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFALWAYS-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP6]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; TFALWAYS-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; TFALWAYS-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TFALWAYS-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] ; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; TFALWAYS-NEXT: [[TMP10:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -546,6 +546,8 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 { ; TFFALLBACK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TFFALLBACK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TFFALLBACK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TFFALLBACK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; TFFALLBACK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; TFFALLBACK-NEXT: br label [[VECTOR_BODY:%.*]] ; TFFALLBACK: vector.body: @@ -556,8 +558,6 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 { ; TFFALLBACK-NEXT: [[TMP6:%.*]] = call @foo_vector( [[WIDE_MASKED_LOAD]], [[ACTIVE_LANE_MASK]]) ; TFFALLBACK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFFALLBACK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP6]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; TFFALLBACK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; TFFALLBACK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TFFALLBACK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] ; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; TFFALLBACK-NEXT: [[TMP10:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -599,6 +599,8 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub ; TFNONE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; TFNONE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; TFNONE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 ; TFNONE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, double [[M:%.*]], i64 0 ; TFNONE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -613,8 +615,6 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub ; TFNONE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFNONE-NEXT: store [[TMP7]], ptr [[TMP8]], align 8 ; TFNONE-NEXT: [[TMP9]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], [[TMP5]]) -; TFNONE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; TFNONE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 ; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; TFNONE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TFNONE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -651,6 +651,8 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub ; TFALWAYS-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TFALWAYS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TFALWAYS-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TFALWAYS-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; TFALWAYS-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 2 ; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; TFALWAYS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, double [[M:%.*]], i64 0 ; TFALWAYS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer @@ -668,8 +670,6 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub ; TFALWAYS-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP8]], ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]]) ; TFALWAYS-NEXT: [[TMP10:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP6]], shufflevector ( insertelement ( poison, double -0.000000e+00, i64 0), poison, zeroinitializer) ; TFALWAYS-NEXT: [[TMP11]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], [[TMP10]]) -; TFALWAYS-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; TFALWAYS-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 2 ; TFALWAYS-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]] ; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; TFALWAYS-NEXT: [[TMP14:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -688,6 +688,8 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub ; TFFALLBACK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TFFALLBACK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TFFALLBACK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TFFALLBACK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; TFFALLBACK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 2 ; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; TFFALLBACK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, double [[M:%.*]], i64 0 ; TFFALLBACK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer @@ -705,8 +707,6 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub ; TFFALLBACK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP8]], ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]]) ; TFFALLBACK-NEXT: [[TMP10:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP6]], shufflevector ( insertelement ( poison, double -0.000000e+00, i64 0), poison, zeroinitializer) ; TFFALLBACK-NEXT: [[TMP11]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], [[TMP10]]) -; TFFALLBACK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; TFFALLBACK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 2 ; TFFALLBACK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]] ; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; TFFALLBACK-NEXT: [[TMP14:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll index ca3c935669e97880a0695a698469ad72a278f939..59a1e108b92f00d2c2a0bc6debd5b175d12ec47c 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll @@ -18,6 +18,8 @@ define void @foo() { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 ; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -49,8 +51,6 @@ define void @foo() { ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4f32.nxv4p0( [[VEC_PHI5]], [[TMP10]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: [[TMP16:%.*]] = add nuw nsw [[VEC_IND]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq [[TMP16]], shufflevector ( insertelement ( poison, i64 1024, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll index 3f6d1b70f891c2128d4af5d2e23a9ba1d6d201fe..9bb941739852633aba2086d83eb4646008876eb2 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll @@ -17,28 +17,28 @@ define void @test_invar_gep(ptr %dst) #0 { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv4i64() +; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[INDEX]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer -; CHECK-NEXT: [[TMP5:%.*]] = add zeroinitializer, [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP7:%.*]] = add [[DOTSPLAT]], [[TMP6]] -; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 3 -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 0 -; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP14:%.*]] = mul i32 [[TMP13]], 4 -; CHECK-NEXT: [[TMP15:%.*]] = sub i32 [[TMP14]], 1 -; CHECK-NEXT: [[TMP16:%.*]] = extractelement [[TMP7]], i32 [[TMP15]] -; CHECK-NEXT: store i64 [[TMP16]], ptr [[TMP12]], align 1 -; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[TMP17]], 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]] +; CHECK-NEXT: [[TMP7:%.*]] = add zeroinitializer, [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = mul [[TMP7]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP9:%.*]] = add [[DOTSPLAT]], [[TMP8]] +; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 0 +; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP15]], 4 +; CHECK-NEXT: [[TMP17:%.*]] = sub i32 [[TMP16]], 1 +; CHECK-NEXT: [[TMP18:%.*]] = extractelement [[TMP9]], i32 [[TMP17]] +; CHECK-NEXT: store i64 [[TMP18]], ptr [[TMP14]], align 1 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll index db97dc67967383137ed7556a06b0c31f4a8392a1..a3c108bca140bfec61a32206762c0d649c03635a 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll @@ -24,33 +24,33 @@ define void @test_no_scalarization(ptr %a, ptr noalias %b, i32 %idx, i32 %n) #0 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], [[TMP5]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]] ; CHECK-NEXT: [[IND_END:%.*]] = add i32 [[IDX]], [[N_VEC]] +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP7:%.*]] = mul i32 [[TMP6]], 2 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i32 [[IDX]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer -; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv2i32() -; CHECK-NEXT: [[TMP7:%.*]] = add [[TMP6]], zeroinitializer -; CHECK-NEXT: [[TMP8:%.*]] = mul [[TMP7]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[INDUCTION:%.*]] = add [[DOTSPLAT]], [[TMP8]] -; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP10:%.*]] = mul i32 [[TMP9]], 2 -; CHECK-NEXT: [[TMP11:%.*]] = mul i32 1, [[TMP10]] -; CHECK-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP11]], i64 0 +; CHECK-NEXT: [[TMP8:%.*]] = call @llvm.experimental.stepvector.nxv2i32() +; CHECK-NEXT: [[TMP9:%.*]] = add [[TMP8]], zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = mul [[TMP9]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[INDUCTION:%.*]] = add [[DOTSPLAT]], [[TMP10]] +; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP12:%.*]] = mul i32 [[TMP11]], 2 +; CHECK-NEXT: [[TMP13:%.*]] = mul i32 1, [[TMP12]] +; CHECK-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP13]], i64 0 ; CHECK-NEXT: [[DOTSPLAT2:%.*]] = shufflevector [[DOTSPLATINSERT1]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i32 [[IDX]], [[INDEX]] -; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[OFFSET_IDX]], 0 -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[A:%.*]], [[VEC_IND]] -; CHECK-NEXT: [[TMP14:%.*]] = extractelement [[TMP13]], i32 0 -; CHECK-NEXT: [[TMP15:%.*]] = getelementptr double, ptr [[TMP14]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP15]], align 8 -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[B:%.*]], i32 [[TMP12]] +; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[OFFSET_IDX]], 0 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[A:%.*]], [[VEC_IND]] +; CHECK-NEXT: [[TMP16:%.*]] = extractelement [[TMP15]], i32 0 ; CHECK-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[TMP16]], i32 0 -; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP17]], align 8 -; CHECK-NEXT: [[TMP18:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[TMP18]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP19]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP17]], align 8 +; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[B:%.*]], i32 [[TMP14]] +; CHECK-NEXT: [[TMP19:%.*]] = getelementptr double, ptr [[TMP18]], i32 0 +; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP19]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT2]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -58,7 +58,7 @@ define void @test_no_scalarization(ptr %a, ptr noalias %b, i32 %idx, i32 %n) #0 ; CHECK-NEXT: [[TMP21:%.*]] = call i32 @llvm.vscale.i32() ; CHECK-NEXT: [[TMP22:%.*]] = mul i32 [[TMP21]], 2 ; CHECK-NEXT: [[TMP23:%.*]] = sub i32 [[TMP22]], 1 -; CHECK-NEXT: [[TMP24:%.*]] = extractelement [[TMP13]], i32 [[TMP23]] +; CHECK-NEXT: [[TMP24:%.*]] = extractelement [[TMP15]], i32 [[TMP23]] ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[L_EXIT:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll index 52feff81914a3725c9026d6d35adc6cdacc2874e..43220d582c4c83bff8be1f87f095f10c9d342e46 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll @@ -14,24 +14,24 @@ define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[COND:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = fcmp une [[WIDE_LOAD]], shufflevector ( insertelement ( poison, float 2.000000e+00, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[TMP8]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr [[TMP9]], i32 4, [[TMP7]], poison) -; CHECK-NEXT: [[TMP10:%.*]] = select fast [[TMP7]], [[WIDE_MASKED_LOAD]], zeroinitializer -; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, [[TMP10]]) -; CHECK-NEXT: [[TMP12]] = fadd fast float [[TMP11]], [[VEC_PHI]] -; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP14]] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[COND:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 4 +; CHECK-NEXT: [[TMP9:%.*]] = fcmp une [[WIDE_LOAD]], shufflevector ( insertelement ( poison, float 2.000000e+00, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr float, ptr [[TMP10]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr [[TMP11]], i32 4, [[TMP9]], poison) +; CHECK-NEXT: [[TMP12:%.*]] = select fast [[TMP9]], [[WIDE_MASKED_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, [[TMP12]]) +; CHECK-NEXT: [[TMP14]] = fadd fast float [[TMP13]], [[VEC_PHI]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -39,7 +39,7 @@ define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 1.000000e+00, [[ENTRY]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 1.000000e+00, [[ENTRY]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[INDVARS:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_NEXT:%.*]], [[FOR_INC:%.*]] ] @@ -57,9 +57,9 @@ define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture ; CHECK-NEXT: [[RES]] = phi float [ [[FADD]], [[IF_THEN]] ], [ [[RDX]], [[FOR_BODY]] ] ; CHECK-NEXT: [[INDVARS_NEXT]] = add nuw nsw i64 [[INDVARS]], 1 ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: for.end: -; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ] ; CHECK-NEXT: ret float [[RES_LCSSA]] ; entry: @@ -101,25 +101,25 @@ define float @cond_cmp_sel(ptr noalias %a, ptr noalias %cond, i64 %N) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[RDX_MINMAX_SELECT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[COND:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = fcmp une [[WIDE_LOAD]], shufflevector ( insertelement ( poison, float 3.000000e+00, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[TMP8]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr [[TMP9]], i32 4, [[TMP7]], poison) -; CHECK-NEXT: [[TMP10:%.*]] = select fast [[TMP7]], [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, float 0x7FF0000000000000, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.vector.reduce.fmin.nxv4f32( [[TMP10]]) -; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp fast olt float [[TMP11]], [[VEC_PHI]] -; CHECK-NEXT: [[RDX_MINMAX_SELECT]] = select fast i1 [[RDX_MINMAX_CMP]], float [[TMP11]], float [[VEC_PHI]] -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[COND:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 4 +; CHECK-NEXT: [[TMP9:%.*]] = fcmp une [[WIDE_LOAD]], shufflevector ( insertelement ( poison, float 3.000000e+00, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr float, ptr [[TMP10]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr [[TMP11]], i32 4, [[TMP9]], poison) +; CHECK-NEXT: [[TMP12:%.*]] = select fast [[TMP9]], [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, float 0x7FF0000000000000, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fmin.nxv4f32( [[TMP12]]) +; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp fast olt float [[TMP13]], [[VEC_PHI]] +; CHECK-NEXT: [[RDX_MINMAX_SELECT]] = select fast i1 [[RDX_MINMAX_CMP]], float [[TMP13]], float [[VEC_PHI]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll index 2e994838ff2411b7fc907d37f3b03fc554c6b491..cdc50c57b947fb2f29a4218cf8940292b98336e0 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll @@ -42,6 +42,8 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 8 ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-UNORDERED-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 8 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UNORDERED: vector.body: ; CHECK-UNORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -51,8 +53,6 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i32 0 ; CHECK-UNORDERED-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; CHECK-UNORDERED-NEXT: [[TMP7]] = fadd [[WIDE_LOAD]], [[VEC_PHI]] -; CHECK-UNORDERED-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 8 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; CHECK-UNORDERED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -89,6 +89,8 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 8 ; CHECK-ORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-ORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-ORDERED-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 8 ; CHECK-ORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ORDERED: vector.body: ; CHECK-ORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -98,8 +100,6 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i32 0 ; CHECK-ORDERED-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; CHECK-ORDERED-NEXT: [[TMP7]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], [[WIDE_LOAD]]) -; CHECK-ORDERED-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 8 ; CHECK-ORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; CHECK-ORDERED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-ORDERED-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -136,6 +136,8 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP4]] ; CHECK-ORDERED-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-ORDERED-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 8 ; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 8 ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -153,8 +155,6 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv8f32.p0(ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-ORDERED-TF-NEXT: [[TMP13:%.*]] = select [[ACTIVE_LANE_MASK]], [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP14]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], [[TMP13]]) -; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 8 ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP16]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -230,6 +230,8 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-UNORDERED-NEXT: [[TMP38:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP39:%.*]] = mul i64 [[TMP38]], 32 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UNORDERED: vector.body: ; CHECK-UNORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -275,8 +277,6 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP35]] = fadd [[WIDE_LOAD4]], [[VEC_PHI1]] ; CHECK-UNORDERED-NEXT: [[TMP36]] = fadd [[WIDE_LOAD5]], [[VEC_PHI2]] ; CHECK-UNORDERED-NEXT: [[TMP37]] = fadd [[WIDE_LOAD6]], [[VEC_PHI3]] -; CHECK-UNORDERED-NEXT: [[TMP38:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP39:%.*]] = mul i64 [[TMP38]], 32 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP39]] ; CHECK-UNORDERED-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP40]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -316,6 +316,8 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-ORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-ORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-ORDERED-NEXT: [[TMP38:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-NEXT: [[TMP39:%.*]] = mul i64 [[TMP38]], 32 ; CHECK-ORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ORDERED: vector.body: ; CHECK-ORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -358,8 +360,6 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP35:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP34]], [[WIDE_LOAD1]]) ; CHECK-ORDERED-NEXT: [[TMP36:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP35]], [[WIDE_LOAD2]]) ; CHECK-ORDERED-NEXT: [[TMP37]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP36]], [[WIDE_LOAD3]]) -; CHECK-ORDERED-NEXT: [[TMP38:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-NEXT: [[TMP39:%.*]] = mul i64 [[TMP38]], 32 ; CHECK-ORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP39]] ; CHECK-ORDERED-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-ORDERED-NEXT: br i1 [[TMP40]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -396,6 +396,8 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP4]] ; CHECK-ORDERED-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-ORDERED-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-ORDERED-TF-NEXT: [[TMP69:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-TF-NEXT: [[TMP70:%.*]] = mul i64 [[TMP69]], 32 ; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 32 ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -479,8 +481,6 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP66:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP64]], [[TMP65]]) ; CHECK-ORDERED-TF-NEXT: [[TMP67:%.*]] = select [[ACTIVE_LANE_MASK8]], [[WIDE_MASKED_LOAD11]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP68]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP66]], [[TMP67]]) -; CHECK-ORDERED-TF-NEXT: [[TMP69:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP70:%.*]] = mul i64 [[TMP69]], 32 ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP70]] ; CHECK-ORDERED-TF-NEXT: [[TMP71:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP72:%.*]] = mul i64 [[TMP71]], 8 @@ -589,6 +589,8 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] ; CHECK-UNORDERED-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 +; CHECK-UNORDERED-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 ; CHECK-UNORDERED-NEXT: [[TMP7:%.*]] = insertelement shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer), float [[A2]], i32 0 ; CHECK-UNORDERED-NEXT: [[TMP8:%.*]] = insertelement shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer), float [[A1]], i32 0 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] @@ -606,8 +608,6 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali ; CHECK-UNORDERED-NEXT: [[TMP13:%.*]] = extractvalue { , } [[STRIDED_VEC]], 1 ; CHECK-UNORDERED-NEXT: [[TMP14]] = fadd [[TMP12]], [[VEC_PHI1]] ; CHECK-UNORDERED-NEXT: [[TMP15]] = fadd [[TMP13]], [[VEC_PHI]] -; CHECK-UNORDERED-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]] ; CHECK-UNORDERED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -661,6 +661,8 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali ; CHECK-ORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]] ; CHECK-ORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] ; CHECK-ORDERED-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 +; CHECK-ORDERED-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 4 ; CHECK-ORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ORDERED: vector.body: ; CHECK-ORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -676,8 +678,6 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali ; CHECK-ORDERED-NEXT: [[TMP11:%.*]] = extractvalue { , } [[STRIDED_VEC]], 1 ; CHECK-ORDERED-NEXT: [[TMP12]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[TMP11]]) ; CHECK-ORDERED-NEXT: [[TMP13]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI1]], [[TMP10]]) -; CHECK-ORDERED-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 4 ; CHECK-ORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] ; CHECK-ORDERED-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-ORDERED-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -730,6 +730,8 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali ; CHECK-ORDERED-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP4]] ; CHECK-ORDERED-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] ; CHECK-ORDERED-TF-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 +; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 4 ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; CHECK-ORDERED-TF-NEXT: [[TMP10:%.*]] = sub i64 [[TMP2]], [[TMP9]] @@ -755,8 +757,6 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali ; CHECK-ORDERED-TF-NEXT: [[TMP19]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[TMP18]]) ; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP16]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP21]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI1]], [[TMP20]]) -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 4 ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP23]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP12]]) ; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -867,6 +867,8 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; CHECK-UNORDERED-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 4 ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-UNORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UNORDERED: vector.body: ; CHECK-UNORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -880,8 +882,6 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; CHECK-UNORDERED-NEXT: [[WIDE_LOAD1:%.*]] = load , ptr [[TMP9]], align 4 ; CHECK-UNORDERED-NEXT: [[TMP10:%.*]] = fadd [[WIDE_LOAD]], [[WIDE_LOAD1]] ; CHECK-UNORDERED-NEXT: [[TMP11]] = fadd [[VEC_PHI]], [[TMP10]] -; CHECK-UNORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] ; CHECK-UNORDERED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -929,6 +929,8 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; CHECK-ORDERED-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 4 ; CHECK-ORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]] ; CHECK-ORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-ORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-ORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ORDERED: vector.body: ; CHECK-ORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -942,8 +944,6 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; CHECK-ORDERED-NEXT: [[WIDE_LOAD1:%.*]] = load , ptr [[TMP9]], align 4 ; CHECK-ORDERED-NEXT: [[TMP10:%.*]] = fadd [[WIDE_LOAD]], [[WIDE_LOAD1]] ; CHECK-ORDERED-NEXT: [[TMP11]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[TMP10]]) -; CHECK-ORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-ORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] ; CHECK-ORDERED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-ORDERED-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -991,6 +991,8 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; CHECK-ORDERED-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP5]] ; CHECK-ORDERED-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP2]] ; CHECK-ORDERED-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 4 ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = sub i64 [[N]], [[TMP7]] @@ -1012,8 +1014,6 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = fadd [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]] ; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP16]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP18]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[TMP17]]) -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 4 ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP20]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP10]]) ; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -1110,6 +1110,8 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no ; CHECK-UNORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-UNORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UNORDERED: vector.body: ; CHECK-UNORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1125,8 +1127,6 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no ; CHECK-UNORDERED-NEXT: [[TMP10:%.*]] = xor [[TMP7]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-UNORDERED-NEXT: [[PREDPHI:%.*]] = select [[TMP10]], shufflevector ( insertelement ( poison, float 3.000000e+00, i64 0), poison, zeroinitializer), [[WIDE_MASKED_LOAD]] ; CHECK-UNORDERED-NEXT: [[TMP11]] = fadd [[VEC_PHI]], [[PREDPHI]] -; CHECK-UNORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] ; CHECK-UNORDERED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -1171,6 +1171,8 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no ; CHECK-ORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-ORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-ORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-ORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-ORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ORDERED: vector.body: ; CHECK-ORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1186,8 +1188,6 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no ; CHECK-ORDERED-NEXT: [[TMP10:%.*]] = xor [[TMP7]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-NEXT: [[PREDPHI:%.*]] = select [[TMP10]], shufflevector ( insertelement ( poison, float 3.000000e+00, i64 0), poison, zeroinitializer), [[WIDE_MASKED_LOAD]] ; CHECK-ORDERED-NEXT: [[TMP11]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[PREDPHI]]) -; CHECK-ORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-ORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] ; CHECK-ORDERED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-ORDERED-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -1232,6 +1232,8 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no ; CHECK-ORDERED-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP4]] ; CHECK-ORDERED-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-ORDERED-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 4 ; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -1258,8 +1260,6 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no ; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = or [[TMP15]], [[TMP18]] ; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = select [[TMP19]], [[PREDPHI]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP21]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[TMP20]]) -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 4 ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP23]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -1358,6 +1358,8 @@ define float @fadd_multiple(ptr noalias nocapture %a, ptr noalias nocapture %b, ; CHECK-UNORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 8 ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-UNORDERED-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 8 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UNORDERED: vector.body: ; CHECK-UNORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1371,8 +1373,6 @@ define float @fadd_multiple(ptr noalias nocapture %a, ptr noalias nocapture %b, ; CHECK-UNORDERED-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP8]], i32 0 ; CHECK-UNORDERED-NEXT: [[WIDE_LOAD1:%.*]] = load , ptr [[TMP9]], align 4 ; CHECK-UNORDERED-NEXT: [[TMP10]] = fadd [[TMP7]], [[WIDE_LOAD1]] -; CHECK-UNORDERED-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 8 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] ; CHECK-UNORDERED-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -1498,6 +1498,8 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-UNORDERED-NEXT: [[TMP52:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP53:%.*]] = mul i64 [[TMP52]], 32 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UNORDERED: vector.body: ; CHECK-UNORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1561,8 +1563,6 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP49]] = call @llvm.fmuladd.nxv8f32( [[WIDE_LOAD4]], [[WIDE_LOAD8]], [[VEC_PHI1]]) ; CHECK-UNORDERED-NEXT: [[TMP50]] = call @llvm.fmuladd.nxv8f32( [[WIDE_LOAD5]], [[WIDE_LOAD9]], [[VEC_PHI2]]) ; CHECK-UNORDERED-NEXT: [[TMP51]] = call @llvm.fmuladd.nxv8f32( [[WIDE_LOAD6]], [[WIDE_LOAD10]], [[VEC_PHI3]]) -; CHECK-UNORDERED-NEXT: [[TMP52:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP53:%.*]] = mul i64 [[TMP52]], 32 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP53]] ; CHECK-UNORDERED-NEXT: [[TMP54:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP54]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] @@ -1604,6 +1604,8 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-ORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-ORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-ORDERED-NEXT: [[TMP56:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-NEXT: [[TMP57:%.*]] = mul i64 [[TMP56]], 32 ; CHECK-ORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ORDERED: vector.body: ; CHECK-ORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1668,8 +1670,6 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP53:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP52]], [[TMP49]]) ; CHECK-ORDERED-NEXT: [[TMP54:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP53]], [[TMP50]]) ; CHECK-ORDERED-NEXT: [[TMP55]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP54]], [[TMP51]]) -; CHECK-ORDERED-NEXT: [[TMP56:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-NEXT: [[TMP57:%.*]] = mul i64 [[TMP56]], 32 ; CHECK-ORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP57]] ; CHECK-ORDERED-NEXT: [[TMP58:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-ORDERED-NEXT: br i1 [[TMP58]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] @@ -1708,6 +1708,8 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP4]] ; CHECK-ORDERED-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-ORDERED-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-ORDERED-TF-NEXT: [[TMP87:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-TF-NEXT: [[TMP88:%.*]] = mul i64 [[TMP87]], 32 ; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 32 ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -1813,8 +1815,6 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP84:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP82]], [[TMP83]]) ; CHECK-ORDERED-TF-NEXT: [[TMP85:%.*]] = select [[ACTIVE_LANE_MASK8]], [[TMP78]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP86]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP84]], [[TMP85]]) -; CHECK-ORDERED-TF-NEXT: [[TMP87:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP88:%.*]] = mul i64 [[TMP87]], 32 ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP88]] ; CHECK-ORDERED-TF-NEXT: [[TMP89:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP90:%.*]] = mul i64 [[TMP89]], 8 @@ -1912,6 +1912,8 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-UNORDERED-NEXT: [[TMP52:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP53:%.*]] = mul i64 [[TMP52]], 32 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UNORDERED: vector.body: ; CHECK-UNORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1975,8 +1977,6 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP49]] = call nnan @llvm.fmuladd.nxv8f32( [[WIDE_LOAD4]], [[WIDE_LOAD8]], [[VEC_PHI1]]) ; CHECK-UNORDERED-NEXT: [[TMP50]] = call nnan @llvm.fmuladd.nxv8f32( [[WIDE_LOAD5]], [[WIDE_LOAD9]], [[VEC_PHI2]]) ; CHECK-UNORDERED-NEXT: [[TMP51]] = call nnan @llvm.fmuladd.nxv8f32( [[WIDE_LOAD6]], [[WIDE_LOAD10]], [[VEC_PHI3]]) -; CHECK-UNORDERED-NEXT: [[TMP52:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP53:%.*]] = mul i64 [[TMP52]], 32 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP53]] ; CHECK-UNORDERED-NEXT: [[TMP54:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP54]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] @@ -2018,6 +2018,8 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-ORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-ORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-ORDERED-NEXT: [[TMP56:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-NEXT: [[TMP57:%.*]] = mul i64 [[TMP56]], 32 ; CHECK-ORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ORDERED: vector.body: ; CHECK-ORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -2082,8 +2084,6 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP53:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP52]], [[TMP49]]) ; CHECK-ORDERED-NEXT: [[TMP54:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP53]], [[TMP50]]) ; CHECK-ORDERED-NEXT: [[TMP55]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP54]], [[TMP51]]) -; CHECK-ORDERED-NEXT: [[TMP56:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-NEXT: [[TMP57:%.*]] = mul i64 [[TMP56]], 32 ; CHECK-ORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP57]] ; CHECK-ORDERED-NEXT: [[TMP58:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-ORDERED-NEXT: br i1 [[TMP58]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] @@ -2122,6 +2122,8 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP4]] ; CHECK-ORDERED-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-ORDERED-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-ORDERED-TF-NEXT: [[TMP87:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-TF-NEXT: [[TMP88:%.*]] = mul i64 [[TMP87]], 32 ; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 32 ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -2227,8 +2229,6 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP84:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP82]], [[TMP83]]) ; CHECK-ORDERED-TF-NEXT: [[TMP85:%.*]] = select nnan [[ACTIVE_LANE_MASK8]], [[TMP78]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP86]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP84]], [[TMP85]]) -; CHECK-ORDERED-TF-NEXT: [[TMP87:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP88:%.*]] = mul i64 [[TMP87]], 32 ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP88]] ; CHECK-ORDERED-TF-NEXT: [[TMP89:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP90:%.*]] = mul i64 [[TMP89]], 8 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll index b3de02799f5139711248d58adb6111fa2c892750..7848d0836d546fec38aff6c191bebceca7b9914c 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll @@ -13,21 +13,21 @@ define void @cond_inv_load_i32i32i16(ptr noalias nocapture %a, ptr noalias nocap ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[INV:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP3]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = icmp ne [[WIDE_LOAD]], zeroinitializer -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i16.nxv4p0( [[BROADCAST_SPLAT]], i32 2, [[TMP4]], poison) -; CHECK-NEXT: [[TMP5:%.*]] = sext [[WIDE_MASKED_GATHER]] to -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP5]], ptr [[TMP6]], i32 4, [[TMP4]]) -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP5]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = icmp ne [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i16.nxv4p0( [[BROADCAST_SPLAT]], i32 2, [[TMP6]], poison) +; CHECK-NEXT: [[TMP7:%.*]] = sext [[WIDE_MASKED_GATHER]] to +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP7]], ptr [[TMP8]], i32 4, [[TMP6]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -92,20 +92,20 @@ define void @cond_inv_load_f64f64f64(ptr noalias nocapture %a, ptr noalias nocap ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[INV:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[COND:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP3]], align 8 -; CHECK-NEXT: [[TMP4:%.*]] = fcmp ogt [[WIDE_LOAD]], shufflevector ( insertelement ( poison, double 4.000000e-01, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4f64.nxv4p0( [[BROADCAST_SPLAT]], i32 8, [[TMP4]], poison) -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: call void @llvm.masked.store.nxv4f64.p0( [[WIDE_MASKED_GATHER]], ptr [[TMP5]], i32 8, [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[COND:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP5]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = fcmp ogt [[WIDE_LOAD]], shufflevector ( insertelement ( poison, double 4.000000e-01, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4f64.nxv4p0( [[BROADCAST_SPLAT]], i32 8, [[TMP6]], poison) +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: call void @llvm.masked.store.nxv4f64.p0( [[WIDE_MASKED_GATHER]], ptr [[TMP7]], i32 8, [[TMP6]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: middle.block: @@ -168,24 +168,24 @@ define void @invariant_load_cond(ptr noalias nocapture %a, ptr nocapture readonl ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 42 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[TMP3]], i64 0 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 42 +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[TMP5]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP4]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = icmp ne [[WIDE_LOAD]], zeroinitializer -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP6]], i32 4, [[TMP5]], poison) -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[BROADCAST_SPLAT]], i32 4, [[TMP5]], poison) -; CHECK-NEXT: [[TMP7:%.*]] = add nsw [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_LOAD]] -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP7]], ptr [[TMP8]], i32 4, [[TMP5]]) -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP10:%.*]] = shl nuw nsw i64 [[TMP9]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP8]], i32 4, [[TMP7]], poison) +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[BROADCAST_SPLAT]], i32 4, [[TMP7]], poison) +; CHECK-NEXT: [[TMP9:%.*]] = add nsw [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_LOAD]] +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP9]], ptr [[TMP10]], i32 4, [[TMP7]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll index 04af618014a795cbd967c562b2067986529b084a..982915fe5cc53f6c0e48c01cf2579426f5c8a69c 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll @@ -19,6 +19,8 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -42,8 +44,6 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) { ; CHECK-NEXT: [[TMP17]] = and i64 [[TMP16]], [[VEC_PHI]] ; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vector.reduce.and.nxv2i64( [[WIDE_LOAD3]]) ; CHECK-NEXT: [[TMP19]] = and i64 [[TMP18]], [[VEC_PHI2]] -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP21]] ; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll index f84e7c0ea182ba9a87281ed7955bf0541aaf92ca..f48933a76709d703a67ddace65853194fc944551 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll @@ -19,6 +19,8 @@ define i64 @int_reduction_add(ptr %a, i64 %N) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -40,8 +42,6 @@ define i64 @int_reduction_add(ptr %a, i64 %N) { ; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP15]], align 8 ; CHECK-NEXT: [[TMP16]] = add [[WIDE_LOAD]], [[VEC_PHI]] ; CHECK-NEXT: [[TMP17]] = add [[WIDE_LOAD3]], [[VEC_PHI2]] -; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll index 23fd5fe5e908ec84bd3a7a17aed05db10eb3bcfb..fd3d8d09fb39d251128365a6e78609d640fe8ce0 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll @@ -19,29 +19,29 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 8 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 8 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0xFFFFFFFFE0000000, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 0 -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 1 -; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]] -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP10]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP12]], align 4 -; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 -; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP10]], i64 [[TMP14]] -; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load , ptr [[TMP15]], align 4 -; CHECK-NEXT: [[TMP16:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[WIDE_LOAD]]) -; CHECK-NEXT: [[TMP17]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[TMP16]], [[WIDE_LOAD2]]) -; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0xFFFFFFFFE0000000, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0 +; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP11]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP14]], align 4 +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i64 [[TMP16]] +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load , ptr [[TMP17]], align 4 +; CHECK-NEXT: [[TMP18:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[WIDE_LOAD]]) +; CHECK-NEXT: [[TMP19]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[TMP18]], [[WIDE_LOAD2]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -52,7 +52,7 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) { ; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2 ; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]] ; CHECK: vec.epilog.ph: -; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 0xFFFFFFFFE0000000, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ], [ [[TMP17]], [[VEC_EPILOG_ITER_CHECK]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 0xFFFFFFFFE0000000, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ], [ [[TMP19]], [[VEC_EPILOG_ITER_CHECK]] ] ; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ] ; CHECK-NEXT: [[N_MOD_VF3:%.*]] = urem i64 [[N]], 2 ; CHECK-NEXT: [[N_VEC4:%.*]] = sub i64 [[N]], [[N_MOD_VF3]] @@ -73,7 +73,7 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) { ; CHECK-NEXT: br i1 [[CMP_N5]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]] ; CHECK: vec.epilog.scalar.ph: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ] -; CHECK-NEXT: [[BC_MERGE_RDX10:%.*]] = phi float [ 0xFFFFFFFFE0000000, [[ITER_CHECK]] ], [ [[TMP17]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[BC_MERGE_RDX10:%.*]] = phi float [ 0xFFFFFFFFE0000000, [[ITER_CHECK]] ], [ [[TMP19]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] @@ -85,7 +85,7 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) { ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] ; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: for.end: -; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ], [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ] ; CHECK-NEXT: ret float [[ADD_LCSSA]] ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll index 6982b3f0943ce3e6b66af6496022c41b77572d54..d5f41aa440e5b072bec656dbaa11761b45047e4f 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll @@ -35,26 +35,26 @@ define void @main_vf_vscale_x_16(ptr %A) #0 { ; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 32 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP5]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 32 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 16 -; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0 -; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 1 -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP6]] -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP11]] -; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i32 0 -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP14]], align 1 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 16 -; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 [[TMP16]] -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP17]], align 1 -; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 32 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] +; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 16 +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP10]], 0 +; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 1 +; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP13]] +; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i32 0 +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP16]], align 1 +; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[TMP17]], 16 +; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 [[TMP18]] +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP19]], align 1 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -72,16 +72,16 @@ define void @main_vf_vscale_x_16(ptr %A) #0 { ; CHECK-NEXT: [[TMP24:%.*]] = mul i64 [[TMP23]], 8 ; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 1024, [[TMP24]] ; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 1024, [[N_MOD_VF2]] +; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 8 ; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]] ; CHECK: vec.epilog.vector.body: ; CHECK-NEXT: [[INDEX5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP25:%.*]] = add i64 [[INDEX5]], 0 -; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP25]] -; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[TMP26]], i32 0 -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP27]], align 1 -; CHECK-NEXT: [[TMP28:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP29:%.*]] = mul i64 [[TMP28]], 8 -; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX5]], [[TMP29]] +; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[INDEX5]], 0 +; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP27]] +; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[TMP28]], i32 0 +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP29]], align 1 +; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX5]], [[TMP26]] ; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]] ; CHECK-NEXT: br i1 [[TMP30]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: vec.epilog.middle.block: @@ -113,26 +113,26 @@ define void @main_vf_vscale_x_16(ptr %A) #0 { ; CHECK-VF8-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-VF8-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-VF8-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-VF8-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 32 ; CHECK-VF8-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-VF8: vector.body: ; CHECK-VF8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-VF8-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-VF8-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 16 -; CHECK-VF8-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 0 -; CHECK-VF8-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 1 -; CHECK-VF8-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]] -; CHECK-VF8-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-VF8-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP9]] -; CHECK-VF8-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i32 0 -; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP12]], align 1 -; CHECK-VF8-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 16 -; CHECK-VF8-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 [[TMP14]] -; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP15]], align 1 -; CHECK-VF8-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 32 -; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]] +; CHECK-VF8-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-VF8-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 16 +; CHECK-VF8-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0 +; CHECK-VF8-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 1 +; CHECK-VF8-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-VF8-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-VF8-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP11]] +; CHECK-VF8-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i32 0 +; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP14]], align 1 +; CHECK-VF8-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 16 +; CHECK-VF8-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 [[TMP16]] +; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP17]], align 1 +; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-VF8-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-VF8-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK-VF8: middle.block: @@ -213,26 +213,26 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 0 -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 1 -; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP9]] -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i32 0 -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP12]], align 1 -; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 2 -; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i64 [[TMP14]] -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP15]], align 1 -; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0 +; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP11]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0 +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP14]], align 1 +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i64 [[TMP16]] +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP17]], align 1 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: middle.block: @@ -282,26 +282,26 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) { ; CHECK-VF8-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-VF8-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-VF8-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-VF8-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-VF8-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-VF8: vector.body: ; CHECK-VF8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-VF8-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-VF8-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 -; CHECK-VF8-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 0 -; CHECK-VF8-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 1 -; CHECK-VF8-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]] -; CHECK-VF8-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-VF8-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP9]] -; CHECK-VF8-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i32 0 -; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP12]], align 1 -; CHECK-VF8-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 2 -; CHECK-VF8-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i64 [[TMP14]] -; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP15]], align 1 -; CHECK-VF8-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 -; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]] +; CHECK-VF8-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-VF8-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 +; CHECK-VF8-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0 +; CHECK-VF8-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 1 +; CHECK-VF8-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-VF8-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-VF8-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP11]] +; CHECK-VF8-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0 +; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP14]], align 1 +; CHECK-VF8-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 +; CHECK-VF8-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i64 [[TMP16]] +; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP17]], align 1 +; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-VF8-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-VF8-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK-VF8: middle.block: @@ -372,26 +372,26 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 { ; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 32 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 10000, [[TMP5]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 10000, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 32 ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[N_VEC]] ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]] -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 16 -; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0 -; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]] -; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP10]] -; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0 -; CHECK-NEXT: store zeroinitializer, ptr [[TMP11]], align 1 -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 16 -; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP13]] -; CHECK-NEXT: store zeroinitializer, ptr [[TMP14]], align 1 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 32 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] +; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 16 +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP10]], 0 +; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP12]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0 +; CHECK-NEXT: store zeroinitializer, ptr [[TMP13]], align 1 +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 16 +; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP15]] +; CHECK-NEXT: store zeroinitializer, ptr [[TMP16]], align 1 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: middle.block: @@ -412,16 +412,16 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 { ; CHECK-NEXT: [[N_MOD_VF3:%.*]] = urem i64 10000, [[TMP21]] ; CHECK-NEXT: [[N_VEC4:%.*]] = sub i64 10000, [[N_MOD_VF3]] ; CHECK-NEXT: [[IND_END6:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC4]] +; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 8 ; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]] ; CHECK: vec.epilog.vector.body: ; CHECK-NEXT: [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[INDEX10]], 0 -; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP22]] -; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i32 0 -; CHECK-NEXT: store zeroinitializer, ptr [[TMP23]], align 1 -; CHECK-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP25:%.*]] = mul i64 [[TMP24]], 8 -; CHECK-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX10]], [[TMP25]] +; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[INDEX10]], 0 +; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP24]] +; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i32 0 +; CHECK-NEXT: store zeroinitializer, ptr [[TMP25]], align 1 +; CHECK-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX10]], [[TMP23]] ; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT12]], [[N_VEC4]] ; CHECK-NEXT: br i1 [[TMP26]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] ; CHECK: vec.epilog.middle.block: @@ -455,26 +455,26 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 { ; CHECK-VF8-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-VF8-NEXT: [[N_MOD_VF:%.*]] = urem i64 10000, [[TMP3]] ; CHECK-VF8-NEXT: [[N_VEC:%.*]] = sub i64 10000, [[N_MOD_VF]] +; CHECK-VF8-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 32 ; CHECK-VF8-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[N_VEC]] ; CHECK-VF8-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-VF8: vector.body: ; CHECK-VF8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-VF8-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-VF8-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]] -; CHECK-VF8-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 16 -; CHECK-VF8-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 0 -; CHECK-VF8-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], [[TMP7]] -; CHECK-VF8-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]] -; CHECK-VF8-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0 -; CHECK-VF8-NEXT: store zeroinitializer, ptr [[TMP9]], align 1 -; CHECK-VF8-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 16 -; CHECK-VF8-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP11]] -; CHECK-VF8-NEXT: store zeroinitializer, ptr [[TMP12]], align 1 -; CHECK-VF8-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 32 -; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP14]] +; CHECK-VF8-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-VF8-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]] +; CHECK-VF8-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 16 +; CHECK-VF8-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0 +; CHECK-VF8-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]] +; CHECK-VF8-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP10]] +; CHECK-VF8-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0 +; CHECK-VF8-NEXT: store zeroinitializer, ptr [[TMP11]], align 1 +; CHECK-VF8-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 16 +; CHECK-VF8-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP13]] +; CHECK-VF8-NEXT: store zeroinitializer, ptr [[TMP14]], align 1 +; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-VF8-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-VF8-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK-VF8: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll index 995950e46f533f3b15969a6f0c9fb83bd570fc74..061b59f5442cdb3a1d8f108846ef9e4189d908f4 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll @@ -31,29 +31,29 @@ define void @fneg(ptr nocapture noundef writeonly %d, ptr nocapture noundef read ; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 16 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP7]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 16 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds half, ptr [[S]], i64 [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds half, ptr [[TMP9]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP10]], align 2 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 8 -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds half, ptr [[TMP9]], i64 [[TMP12]] -; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP13]], align 2 -; CHECK-NEXT: [[TMP14:%.*]] = fneg [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP15:%.*]] = fneg [[WIDE_LOAD3]] -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds half, ptr [[D]], i64 [[TMP8]] -; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds half, ptr [[TMP16]], i32 0 -; CHECK-NEXT: store [[TMP14]], ptr [[TMP17]], align 2 -; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 8 -; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds half, ptr [[TMP16]], i64 [[TMP19]] -; CHECK-NEXT: store [[TMP15]], ptr [[TMP20]], align 2 -; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 16 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP22]] +; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds half, ptr [[S]], i64 [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds half, ptr [[TMP11]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP12]], align 2 +; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 8 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds half, ptr [[TMP11]], i64 [[TMP14]] +; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP15]], align 2 +; CHECK-NEXT: [[TMP16:%.*]] = fneg [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP17:%.*]] = fneg [[WIDE_LOAD3]] +; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds half, ptr [[D]], i64 [[TMP10]] +; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds half, ptr [[TMP18]], i32 0 +; CHECK-NEXT: store [[TMP16]], ptr [[TMP19]], align 2 +; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 8 +; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds half, ptr [[TMP18]], i64 [[TMP21]] +; CHECK-NEXT: store [[TMP17]], ptr [[TMP22]], align 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll index 3a867cf401b3390163d2a0c1dac4fab3f1206af6..a9657971d48aedc35ed96da56d340ad084e3a0b7 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll @@ -13,18 +13,18 @@ define void @gather_nxv4i32_ind64(ptr noalias nocapture readonly %a, ptr noalias ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP3]], align 8 -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], [[WIDE_LOAD]] -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4f32.nxv4p0( [[TMP4]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[C:%.*]], i64 [[INDEX]] -; CHECK-NEXT: store [[WIDE_MASKED_GATHER]], ptr [[TMP5]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP5]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], [[WIDE_LOAD]] +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4f32.nxv4p0( [[TMP6]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[C:%.*]], i64 [[INDEX]] +; CHECK-NEXT: store [[WIDE_MASKED_GATHER]], ptr [[TMP7]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -80,19 +80,19 @@ define void @scatter_nxv4i32_ind32(ptr noalias nocapture %a, ptr noalias nocaptu ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[C:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP3]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load , ptr [[TMP4]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = sext [[WIDE_LOAD1]] to -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], [[TMP5]] -; CHECK-NEXT: call void @llvm.masked.scatter.nxv4f32.nxv4p0( [[WIDE_LOAD]], [[TMP6]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[C:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP5]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load , ptr [[TMP6]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = sext [[WIDE_LOAD1]] to +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], [[TMP7]] +; CHECK-NEXT: call void @llvm.masked.scatter.nxv4f32.nxv4p0( [[WIDE_LOAD]], [[TMP8]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: middle.block: @@ -147,18 +147,18 @@ define void @scatter_inv_nxv4i32(ptr noalias nocapture %inv, ptr noalias nocaptu ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[INV:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP3]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = icmp ne [[WIDE_LOAD]], zeroinitializer -; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( shufflevector ( insertelement ( poison, i32 3, i64 0), poison, zeroinitializer), [[BROADCAST_SPLAT]], i32 4, [[TMP4]]) -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[TMP5]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP5]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = icmp ne [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( shufflevector ( insertelement ( poison, i32 3, i64 0), poison, zeroinitializer), [[BROADCAST_SPLAT]], i32 4, [[TMP6]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK: middle.block: @@ -217,19 +217,19 @@ define void @gather_inv_nxv4i32(ptr noalias nocapture %a, ptr noalias nocapture ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[INV:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP3]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = icmp sgt [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 3, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[BROADCAST_SPLAT]], i32 4, [[TMP4]], poison) -; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[WIDE_MASKED_GATHER]], ptr [[TMP3]], i32 4, [[TMP4]]) -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[TMP5]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP5]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 3, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[BROADCAST_SPLAT]], i32 4, [[TMP6]], poison) +; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[WIDE_MASKED_GATHER]], ptr [[TMP5]], i32 4, [[TMP6]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] ; CHECK: middle.block: @@ -292,31 +292,31 @@ define void @gather_nxv4i32_ind64_stride2(ptr noalias nocapture %a, ptr noalias ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -8 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] -; CHECK-NEXT: [[TMP3:%.*]] = call @llvm.experimental.stepvector.nxv4i64() -; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2 -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 3 +; CHECK-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv4i64() +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP7]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[STEP_ADD:%.*]] = add [[VEC_IND]], [[DOTSPLAT]] -; CHECK-NEXT: [[TMP6:%.*]] = shl [[VEC_IND]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP7:%.*]] = shl [[STEP_ADD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], [[TMP6]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[B]], [[TMP7]] -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4f32.nxv4p0( [[TMP8]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) -; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call @llvm.masked.gather.nxv4f32.nxv4p0( [[TMP9]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: store [[WIDE_MASKED_GATHER]], ptr [[TMP10]], align 4 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = shl nuw nsw i64 [[TMP11]], 2 -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP10]], i64 [[TMP12]] -; CHECK-NEXT: store [[WIDE_MASKED_GATHER2]], ptr [[TMP13]], align 4 -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 3 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] +; CHECK-NEXT: [[TMP8:%.*]] = shl [[VEC_IND]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP9:%.*]] = shl [[STEP_ADD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], [[TMP8]] +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[B]], [[TMP9]] +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4f32.nxv4p0( [[TMP10]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) +; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call @llvm.masked.gather.nxv4f32.nxv4p0( [[TMP11]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: store [[WIDE_MASKED_GATHER]], ptr [[TMP12]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP14:%.*]] = shl nuw nsw i64 [[TMP13]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i64 [[TMP14]] +; CHECK-NEXT: store [[WIDE_MASKED_GATHER2]], ptr [[TMP15]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[STEP_ADD]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll index 01657493e9d4a662fd6488699cddbe7a129baffc..0fe8fa3f4154a22e887d348c3b0aacd67e00553f 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll @@ -11,7 +11,10 @@ target triple = "aarch64-unknown-linux-gnu" define void @induction_i7(ptr %dst) #0 { ; CHECK-LABEL: @induction_i7( ; CHECK: vector.ph: -; CHECK: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i8() +; CHECK: %ind.end = trunc i64 %n.vec to i7 +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 +; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i8() ; CHECK: [[TMP5:%.*]] = trunc [[TMP4]] to ; CHECK-NEXT: [[TMP6:%.*]] = add [[TMP5]], zeroinitializer ; CHECK-NEXT: [[TMP7:%.*]] = mul [[TMP6]], shufflevector ( insertelement ( poison, i7 1, i64 0), poison, zeroinitializer) @@ -25,8 +28,6 @@ define void @induction_i7(ptr %dst) #0 { ; CHECK-NEXT: [[EXT:%.+]] = zext [[TMP11]] to ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0 ; CHECK-NEXT: store [[EXT]], ptr [[TMP13]], align 8 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], ; @@ -57,6 +58,9 @@ for.end: ; preds = %for.body define void @induction_i3_zext(ptr %dst) #0 { ; CHECK-LABEL: @induction_i3_zext( ; CHECK: vector.ph: +; CHECK: %ind.end = trunc i64 %n.vec to i3 +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 ; CHECK: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i8() ; CHECK: [[TMP5:%.*]] = trunc [[TMP4]] to ; CHECK-NEXT: [[TMP6:%.*]] = add [[TMP5]], zeroinitializer @@ -70,8 +74,6 @@ define void @induction_i3_zext(ptr %dst) #0 { ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DST:%.*]], i64 [[TMP9]] ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0 ; CHECK-NEXT: store [[TMP10]], ptr [[TMP13]], align 8 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], ; diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll index 739010b5b631f9acaebb2a0e8b569eb6d50edc67..965c71c008aa19f9a6935da441c82fc2f9206908 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll @@ -23,26 +23,26 @@ define void @cond_ind64(ptr noalias nocapture %a, ptr noalias nocapture readonly ; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] -; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv4i64() -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP6]], i64 0 +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[TMP4]], 2 +; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv4i64() +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[TMP7]], 2 +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP8]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP7:%.*]] = trunc [[VEC_IND]] to -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[B:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP8]], i32 4, [[TMP7]], poison) -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[WIDE_MASKED_LOAD]], ptr [[TMP10]], i32 4, [[TMP7]]) -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = shl i64 [[TMP12]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP9:%.*]] = trunc [[VEC_IND]] to +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[B:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP10]], i32 4, [[TMP9]], poison) +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[WIDE_MASKED_LOAD]], ptr [[TMP11]], i32 4, [[TMP9]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] -; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] @@ -56,14 +56,14 @@ define void @cond_ind64(ptr noalias nocapture %a, ptr noalias nocapture readonly ; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label [[FOR_INC]], label [[IF_THEN:%.*]] ; CHECK: if.then: ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I_08]] -; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 ; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_08]] -; CHECK-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX1]], align 4 +; CHECK-NEXT: store i32 [[TMP13]], ptr [[ARRAYIDX1]], align 4 ; CHECK-NEXT: br label [[FOR_INC]] ; CHECK: for.inc: ; CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_08]], 1 ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] -; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: exit: ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll index 0b5058cff8d5c8bb8cd13b10139ef3c3806fcd0b..f600a0d5877d33b2b660a0f14cf00a192305c0ad 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll @@ -26,6 +26,8 @@ define void @test_array_load2_store2(i32 %C, i32 %D) #1 { ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[C:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[D:%.*]], i64 0 @@ -46,8 +48,6 @@ define void @test_array_load2_store2(i32 %C, i32 %D) #1 { ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 -1 ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call @llvm.experimental.vector.interleave2.nxv8i32( [[TMP4]], [[TMP5]]) ; CHECK-NEXT: store [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512 ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -106,6 +106,8 @@ define void @test_array_load2_i16_store2(i32 %C, i32 %D) #1 { ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[TMP0:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP1:%.*]] = shl [[TMP0]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() @@ -134,8 +136,6 @@ define void @test_array_load2_i16_store2(i32 %C, i32 %D) #1 { ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i64 -1 ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call @llvm.experimental.vector.interleave2.nxv8i32( [[TMP8]], [[TMP10]]) ; CHECK-NEXT: store [[INTERLEAVED_VEC]], ptr [[TMP13]], align 4 -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512 @@ -197,6 +197,8 @@ define void @test_array_load2_store2_i16(i32 noundef %C, i32 noundef %D) #1 { ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[TMP0:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP1:%.*]] = shl [[TMP0]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() @@ -226,8 +228,6 @@ define void @test_array_load2_store2_i16(i32 noundef %C, i32 noundef %D) #1 { ; CHECK-NEXT: [[TMP12:%.*]] = trunc [[TMP11]] to ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [1024 x i16], ptr @CD_i16, i64 0, [[TMP7]] ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i16.nxv4p0( [[TMP12]], [[TMP13]], i32 2, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512 @@ -277,6 +277,8 @@ define i32 @test_struct_load6(%struct.ST6* %S) #1 { ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 2 ; CHECK-NEXT: [[TMP0:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2 @@ -305,8 +307,6 @@ define i32 @test_struct_load6(%struct.ST6* %S) #1 { ; CHECK-NEXT: [[TMP12:%.*]] = add [[TMP11]], [[WIDE_MASKED_GATHER4]] ; CHECK-NEXT: [[TMP13:%.*]] = add [[TMP12]], [[WIDE_MASKED_GATHER5]] ; CHECK-NEXT: [[TMP14]] = sub [[TMP10]], [[TMP13]] -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 @@ -381,6 +381,8 @@ define void @test_reversed_load2_store2(%struct.ST2* noalias nocapture readonly ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP19:%.*]] = shl nuw nsw i64 [[TMP18]], 2 ; CHECK-NEXT: [[TMP0:%.*]] = call @llvm.experimental.stepvector.nxv4i32() ; CHECK-NEXT: [[INDUCTION:%.*]] = sub shufflevector ( insertelement ( poison, i32 1023, i64 0), poison, zeroinitializer), [[TMP0]] ; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vscale.i32() @@ -416,8 +418,6 @@ define void @test_reversed_load2_store2(%struct.ST2* noalias nocapture readonly ; CHECK-NEXT: [[REVERSE3:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[TMP11]]) ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call @llvm.experimental.vector.interleave2.nxv8i32( [[REVERSE2]], [[REVERSE3]]) ; CHECK-NEXT: store [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4 -; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP19:%.*]] = shl nuw nsw i64 [[TMP18]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 @@ -475,6 +475,8 @@ define void @even_load_static_tc(i32* noalias nocapture readonly %A, i32* noalia ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2 ; CHECK-NEXT: [[N_VEC:%.*]] = sub nuw nsw i64 512, [[TMP1]] ; CHECK-NEXT: [[IND_END:%.*]] = shl nuw nsw i64 [[N_VEC]], 1 +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -487,8 +489,6 @@ define void @even_load_static_tc(i32* noalias nocapture readonly %A, i32* noalia ; CHECK-NEXT: [[TMP5:%.*]] = and i64 [[INDEX]], 9223372036854775804 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP5]] ; CHECK-NEXT: store [[TMP4]], ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -561,6 +561,8 @@ define void @even_load_dynamic_tc(i32* noalias nocapture readonly %A, i32* noali ; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP6]], i64 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[TMP9]] ; CHECK-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 1 +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -573,8 +575,6 @@ define void @even_load_dynamic_tc(i32* noalias nocapture readonly %A, i32* noali ; CHECK-NEXT: [[TMP13:%.*]] = and i64 [[INDEX]], 9223372036854775804 ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP13]] ; CHECK-NEXT: store [[TMP12]], ptr [[TMP14]], align 4 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] @@ -640,6 +640,8 @@ define void @load_gap_reverse(%pair* noalias nocapture readonly %P1, %pair* noal ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 ; CHECK-NEXT: [[TMP0:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[INDUCTION:%.*]] = sub shufflevector ( insertelement ( poison, i64 1023, i64 0), poison, zeroinitializer), [[TMP0]] ; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() @@ -659,8 +661,6 @@ define void @load_gap_reverse(%pair* noalias nocapture readonly %P1, %pair* noal ; CHECK-NEXT: [[TMP5:%.*]] = sub nsw [[WIDE_MASKED_GATHER]], [[VEC_IND]] ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i64.nxv4p0( [[TMP2]], [[TMP3]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i64.nxv4p0( [[TMP5]], [[TMP4]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 @@ -709,6 +709,8 @@ define void @mixed_load2_store2(i32* noalias nocapture readonly %A, i32* noalias ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -728,8 +730,6 @@ define void @mixed_load2_store2(i32* noalias nocapture readonly %A, i32* noalias ; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 -1 ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call @llvm.experimental.vector.interleave2.nxv8i32( [[TMP4]], [[TMP7]]) ; CHECK-NEXT: store [[INTERLEAVED_VEC]], ptr [[TMP9]], align 4 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512 ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] @@ -802,6 +802,8 @@ define void @int_float_struct(%struct.IntFloat* nocapture readonly %p) #0 { ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -815,8 +817,6 @@ define void @int_float_struct(%struct.IntFloat* nocapture readonly %p) #0 { ; CHECK-NEXT: [[TMP3:%.*]] = bitcast [[TMP2]] to ; CHECK-NEXT: [[TMP4]] = add [[TMP1]], [[VEC_PHI1]] ; CHECK-NEXT: [[TMP5]] = fadd fast [[VEC_PHI]], [[TMP3]] -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] @@ -892,6 +892,8 @@ define void @PR27626_0(%pair.i32 *%p, i32 %z, i64 %n) #1 { ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]] +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[TMP7:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2 @@ -911,8 +913,6 @@ define void @PR27626_0(%pair.i32 *%p, i32 %z, i64 %n) #1 { ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { , } @llvm.experimental.vector.deinterleave2.nxv8i32( [[WIDE_VEC]]) ; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { , } [[STRIDED_VEC]], 0 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[TMP13]], [[TMP11]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -979,6 +979,8 @@ define i32 @PR27626_1(%pair.i32 *%p, i64 %n) #1 { ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]] +; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i64 [[TMP16]], 2 ; CHECK-NEXT: [[TMP7:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2 @@ -1000,8 +1002,6 @@ define i32 @PR27626_1(%pair.i32 *%p, i64 %n) #1 { ; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { , } @llvm.experimental.vector.deinterleave2.nxv8i32( [[WIDE_VEC1]]) ; CHECK-NEXT: [[TMP14:%.*]] = extractvalue { , } [[STRIDED_VEC2]], 0 ; CHECK-NEXT: [[TMP15]] = add [[TMP14]], [[VEC_PHI]] -; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i64 [[TMP16]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1074,6 +1074,8 @@ define void @PR27626_2(%pair.i32 *%p, i64 %n, i32 %z) #1 { ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]] +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[TMP7:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2 @@ -1093,8 +1095,6 @@ define void @PR27626_2(%pair.i32 *%p, i64 %n, i32 %z) #1 { ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { , } @llvm.experimental.vector.deinterleave2.nxv8i32( [[WIDE_VEC]]) ; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { , } [[STRIDED_VEC]], 0 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[TMP13]], [[TMP12]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1164,6 +1164,8 @@ define i32 @PR27626_3(%pair.i32 *%p, i64 %n, i32 %z) #1 { ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]] +; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP18:%.*]] = shl nuw nsw i64 [[TMP17]], 2 ; CHECK-NEXT: [[TMP7:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2 @@ -1186,8 +1188,6 @@ define i32 @PR27626_3(%pair.i32 *%p, i64 %n, i32 %z) #1 { ; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { , } @llvm.experimental.vector.deinterleave2.nxv8i32( [[WIDE_VEC1]]) ; CHECK-NEXT: [[TMP15:%.*]] = extractvalue { , } [[STRIDED_VEC2]], 0 ; CHECK-NEXT: [[TMP16]] = add [[TMP15]], [[VEC_PHI]] -; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP18:%.*]] = shl nuw nsw i64 [[TMP17]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1267,6 +1267,8 @@ define void @PR27626_4(i32 *%a, i32 %x, i32 %y, i32 %z, i64 %n) #1 { ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP5]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], [[DOTNEG]] ; CHECK-NEXT: [[IND_END:%.*]] = shl nuw i64 [[N_VEC]], 1 +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP7:%.*]] = shl [[TMP6]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() @@ -1291,8 +1293,6 @@ define void @PR27626_4(i32 *%a, i32 %x, i32 %y, i32 %z, i64 %n) #1 { ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i64 -1 ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call @llvm.experimental.vector.interleave2.nxv8i32( [[BROADCAST_SPLAT2]], [[BROADCAST_SPLAT4]]) ; CHECK-NEXT: store [[INTERLEAVED_VEC]], ptr [[TMP13]], align 4 -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1363,6 +1363,8 @@ define void @PR27626_5(i32 *%a, i32 %x, i32 %y, i32 %z, i64 %n) #1 { ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], [[DOTNEG]] ; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[N_VEC]], 1 ; CHECK-NEXT: [[IND_END:%.*]] = or disjoint i64 [[TMP6]], 3 +; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i64 [[TMP16]], 2 ; CHECK-NEXT: [[TMP7:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP8:%.*]] = shl [[TMP7]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[INDUCTION:%.*]] = add [[TMP8]], shufflevector ( insertelement ( poison, i64 3, i64 0), poison, zeroinitializer) @@ -1388,8 +1390,6 @@ define void @PR27626_5(i32 *%a, i32 %x, i32 %y, i32 %z, i64 %n) #1 { ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[BROADCAST_SPLAT]], [[TMP14]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[BROADCAST_SPLAT2]], [[TMP15]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[BROADCAST_SPLAT4]], [[TMP13]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i64 [[TMP16]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1473,6 +1473,8 @@ define void @PR34743(i16* %a, i32* %b, i64 %n) #1 { ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP8]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP1]], [[DOTNEG]] ; CHECK-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 1 +; CHECK-NEXT: [[TMP27:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP28:%.*]] = shl nuw nsw i64 [[TMP27]], 2 ; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() ; CHECK-NEXT: [[TMP10:%.*]] = shl nuw nsw i32 [[TMP9]], 2 ; CHECK-NEXT: [[TMP11:%.*]] = add nsw i32 [[TMP10]], -1 @@ -1501,9 +1503,7 @@ define void @PR34743(i16* %a, i32* %b, i64 %n) #1 { ; CHECK-NEXT: [[TMP24:%.*]] = mul nsw [[TMP22]], [[TMP19]] ; CHECK-NEXT: [[TMP25:%.*]] = mul nsw [[TMP24]], [[TMP23]] ; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: store [[TMP25]], ptr [[TMP26]], align 4, !alias.scope [[META37:![0-9]+]], !noalias [[META34]] -; CHECK-NEXT: [[TMP27:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP28:%.*]] = shl nuw nsw i64 [[TMP27]], 2 +; CHECK-NEXT: store [[TMP25]], ptr [[TMP26]], align 4, !alias.scope [[META37:![0-9]+]], !noalias [[META34]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP28]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll index 56e2fe6e15b7076cdbe1ebdbdedfb34024a6fed5..1b5df2c1bfb857e40abc3718f6186c30246dd25d 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll @@ -35,6 +35,8 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no ; SCALAR_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]] ; SCALAR_TAIL_FOLDING-NEXT: [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]] +; SCALAR_TAIL_FOLDING-NEXT: [[TMP19:%.*]] = call i32 @llvm.vscale.i32() +; SCALAR_TAIL_FOLDING-NEXT: [[TMP20:%.*]] = shl i32 [[TMP19]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv16i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = call i32 @llvm.vscale.i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = shl i32 [[TMP5]], 4 @@ -64,8 +66,6 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no ; SCALAR_TAIL_FOLDING-NEXT: [[INTERLEAVED_VEC:%.*]] = call @llvm.experimental.vector.interleave2.nxv32i8( [[TMP14]], [[TMP15]]) ; SCALAR_TAIL_FOLDING-NEXT: [[INTERLEAVED_MASK1:%.*]] = call @llvm.experimental.vector.interleave2.nxv32i1( [[TMP7]], [[TMP7]]) ; SCALAR_TAIL_FOLDING-NEXT: call void @llvm.masked.store.nxv32i8.p0( [[INTERLEAVED_VEC]], ptr [[TMP18]], i32 1, [[INTERLEAVED_MASK1]]) -; SCALAR_TAIL_FOLDING-NEXT: [[TMP19:%.*]] = call i32 @llvm.vscale.i32() -; SCALAR_TAIL_FOLDING-NEXT: [[TMP20:%.*]] = shl i32 [[TMP19]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP20]] ; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; SCALAR_TAIL_FOLDING-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] @@ -111,6 +111,8 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no ; PREDICATED_TAIL_FOLDING-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; PREDICATED_TAIL_FOLDING: vector.ph: ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP19:%.*]] = call i32 @llvm.vscale.i32() +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP20:%.*]] = shl i32 [[TMP19]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP1]]) @@ -146,8 +148,6 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no ; PREDICATED_TAIL_FOLDING-NEXT: [[INTERLEAVED_VEC:%.*]] = call @llvm.experimental.vector.interleave2.nxv32i8( [[TMP14]], [[TMP15]]) ; PREDICATED_TAIL_FOLDING-NEXT: [[INTERLEAVED_MASK1:%.*]] = call @llvm.experimental.vector.interleave2.nxv32i1( [[TMP10]], [[TMP10]]) ; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.store.nxv32i8.p0( [[INTERLEAVED_VEC]], ptr [[TMP18]], i32 1, [[INTERLEAVED_MASK1]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP19:%.*]] = call i32 @llvm.vscale.i32() -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP20:%.*]] = shl i32 [[TMP19]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP20]] ; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP2]]) ; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] @@ -227,6 +227,8 @@ define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr no ; SCALAR_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]] ; SCALAR_TAIL_FOLDING-NEXT: [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]] +; SCALAR_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = call i32 @llvm.vscale.i32() +; SCALAR_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = shl i32 [[TMP14]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv16i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = call i32 @llvm.vscale.i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = shl i32 [[TMP5]], 4 @@ -247,8 +249,6 @@ define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr no ; SCALAR_TAIL_FOLDING-NEXT: [[TMP12:%.*]] = zext nneg [[TMP11]] to ; SCALAR_TAIL_FOLDING-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP12]] ; SCALAR_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( shufflevector ( insertelement ( poison, i8 2, i64 0), poison, zeroinitializer), [[TMP13]], i32 1, [[TMP10]]) -; SCALAR_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = call i32 @llvm.vscale.i32() -; SCALAR_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = shl i32 [[TMP14]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP15]] ; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; SCALAR_TAIL_FOLDING-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] @@ -286,6 +286,8 @@ define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr no ; PREDICATED_TAIL_FOLDING-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; PREDICATED_TAIL_FOLDING: vector.ph: ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = call i32 @llvm.vscale.i32() +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = shl i32 [[TMP14]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP1]]) @@ -312,8 +314,6 @@ define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr no ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP11]] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP13:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP9]], zeroinitializer ; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( shufflevector ( insertelement ( poison, i8 2, i64 0), poison, zeroinitializer), [[TMP12]], i32 1, [[TMP13]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = call i32 @llvm.vscale.i32() -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = shl i32 [[TMP14]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP15]] ; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP2]]) ; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] @@ -389,6 +389,8 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no ; SCALAR_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]] ; SCALAR_TAIL_FOLDING-NEXT: [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]] +; SCALAR_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = call i32 @llvm.vscale.i32() +; SCALAR_TAIL_FOLDING-NEXT: [[TMP16:%.*]] = shl i32 [[TMP15]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv16i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = call i32 @llvm.vscale.i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = shl i32 [[TMP5]], 4 @@ -412,8 +414,6 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no ; SCALAR_TAIL_FOLDING-NEXT: [[TMP13:%.*]] = zext nneg [[TMP12]] to ; SCALAR_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP13]] ; SCALAR_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( shufflevector ( insertelement ( poison, i8 2, i64 0), poison, zeroinitializer), [[TMP14]], i32 1, [[TMP11]]) -; SCALAR_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = call i32 @llvm.vscale.i32() -; SCALAR_TAIL_FOLDING-NEXT: [[TMP16:%.*]] = shl i32 [[TMP15]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP16]] ; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; SCALAR_TAIL_FOLDING-NEXT: [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] @@ -457,6 +457,8 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no ; PREDICATED_TAIL_FOLDING: vector.ph: ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV3:%.*]] = zext i8 [[GUARD2]] to i32 ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD1]] to i32 +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP16:%.*]] = call i32 @llvm.vscale.i32() +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP17:%.*]] = shl i32 [[TMP16]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP1]]) @@ -487,8 +489,6 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP13]] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP11]], zeroinitializer ; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( shufflevector ( insertelement ( poison, i8 2, i64 0), poison, zeroinitializer), [[TMP14]], i32 1, [[TMP15]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP16:%.*]] = call i32 @llvm.vscale.i32() -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP17:%.*]] = shl i32 [[TMP16]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP17]] ; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP2]]) ; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll index 6a562e8b92bb7a0c64a9dbd7d846c2a55306b320..08da1dedae23e5d35f7676df84dc295b033db8d0 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll @@ -15,21 +15,21 @@ define void @inv_store_i16(ptr noalias %dst, ptr noalias readonly %src, i64 %N) ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[SRC:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 2 -; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP8:%.*]] = mul i32 [[TMP7]], 4 -; CHECK-NEXT: [[TMP9:%.*]] = sub i32 [[TMP8]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = extractelement [[WIDE_LOAD]], i32 [[TMP9]] -; CHECK-NEXT: store i16 [[TMP10]], ptr [[DST:%.*]], align 2 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[SRC:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 2 +; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP10:%.*]] = mul i32 [[TMP9]], 4 +; CHECK-NEXT: [[TMP11:%.*]] = sub i32 [[TMP10]], 1 +; CHECK-NEXT: [[TMP12:%.*]] = extractelement [[WIDE_LOAD]], i32 [[TMP11]] +; CHECK-NEXT: store i16 [[TMP12]], ptr [[DST:%.*]], align 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -45,7 +45,7 @@ define void @inv_store_i16(ptr noalias %dst, ptr noalias readonly %src, i64 %N) ; CHECK-NEXT: store i16 [[LD]], ptr [[DST]], align 2 ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_INC24]], label [[FOR_BODY14]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_INC24]], label [[FOR_BODY14]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: for.inc24: ; CHECK-NEXT: ret void ; @@ -78,20 +78,20 @@ define void @cond_inv_store_i32(ptr noalias %dst, ptr noalias readonly %src, i64 ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[DST:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = icmp sgt [[WIDE_LOAD]], zeroinitializer -; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[WIDE_LOAD]], [[BROADCAST_SPLAT]], i32 4, [[TMP7]]) -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 4 +; CHECK-NEXT: [[TMP9:%.*]] = icmp sgt [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[WIDE_LOAD]], [[BROADCAST_SPLAT]], i32 4, [[TMP9]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll index b378603bbc52ff5bfd6d5aadeaf74c51d4f46edf..d12a1036ed53614c470ffd8e22e3f61e535ca2e0 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll @@ -23,6 +23,8 @@ define ptr @test(ptr %start.1, ptr %start.2, ptr %end) { ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START_1:%.*]], i64 [[TMP8]] ; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[N_VEC]], 8 ; CHECK-NEXT: [[IND_END3:%.*]] = getelementptr i8, ptr [[START_2]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP33:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP34:%.*]] = mul i64 [[TMP33]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START_1]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] @@ -60,8 +62,6 @@ define ptr @test(ptr %start.1, ptr %start.2, ptr %end) { ; CHECK-NEXT: [[TMP31:%.*]] = mul i64 [[TMP30]], 2 ; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i64, ptr [[NEXT_GEP]], i64 [[TMP31]] ; CHECK-NEXT: store zeroinitializer, ptr [[TMP32]], align 8 -; CHECK-NEXT: [[TMP33:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP34:%.*]] = mul i64 [[TMP33]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP34]] ; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP13]] ; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll index 997aca89b825d21cae9f7d00f817f6255561f500..602ccb678c968eea2ebad9d3dcf1909d3ad378b3 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll @@ -4,14 +4,18 @@ target triple = "aarch64-unknown-linux-gnu" define void @trip7_i64(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 { ; CHECK-LABEL: @trip7_i64( +; CHECK: = call i64 @llvm.vscale.i64() +; CHECK-NEXT: = mul i64 +; CHECK: = call i64 @llvm.vscale.i64() +; CHECK-NEXT: = mul i64 +; CHECK: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[VF:%.*]] = mul i64 [[VSCALE]], 2 ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] ; CHECK: [[ACTIVE_LANE_MASK:%.*]] = phi [ {{%.*}}, %vector.ph ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %vector.body ] ; CHECK: {{%.*}} = call @llvm.masked.load.nxv2i64.p0(ptr {{%.*}}, i32 8, [[ACTIVE_LANE_MASK]], poison) ; CHECK: {{%.*}} = call @llvm.masked.load.nxv2i64.p0(ptr {{%.*}}, i32 8, [[ACTIVE_LANE_MASK]], poison) ; CHECK: call void @llvm.masked.store.nxv2i64.p0( {{%.*}}, ptr {{%.*}}, i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[VF:%.*]] = mul i64 [[VSCALE]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[VF]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 7) ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NOT:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll index 79bda0a4a5d804f6696dd6413be939b38d529c72..006171f3b0682e4fa15dc9bf822dc8fb56be7d7a 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll @@ -33,34 +33,34 @@ define void @multiple_exits_unique_exit_block(ptr %A, ptr %B, i32 %N) #0 { ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP10]] +; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP12:%.*]] = mul i32 [[TMP11]], 8 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[INDEX]], 0 -; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP13:%.*]] = mul i32 [[TMP12]], 4 -; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 0 -; CHECK-NEXT: [[TMP15:%.*]] = mul i32 [[TMP14]], 1 -; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[INDEX]], [[TMP15]] -; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP11]] -; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP16]] -; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP19]], align 4 -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i64 [[TMP21]] -; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP22]], align 4 -; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP11]] -; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP16]] -; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i32 0 -; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP25]], align 4 -; CHECK-NEXT: [[TMP26:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP27:%.*]] = mul i64 [[TMP26]], 4 -; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i64 [[TMP27]] -; CHECK-NEXT: store [[WIDE_LOAD3]], ptr [[TMP28]], align 4 -; CHECK-NEXT: [[TMP29:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP30:%.*]] = mul i32 [[TMP29]], 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP30]] +; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[INDEX]], 0 +; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP15:%.*]] = mul i32 [[TMP14]], 4 +; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], 0 +; CHECK-NEXT: [[TMP17:%.*]] = mul i32 [[TMP16]], 1 +; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[INDEX]], [[TMP17]] +; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP13]] +; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP18]] +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 4 +; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 [[TMP23]] +; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP24]], align 4 +; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP13]] +; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP18]] +; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i32 0 +; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP27]], align 4 +; CHECK-NEXT: [[TMP28:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP29:%.*]] = mul i64 [[TMP28]], 4 +; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP29]] +; CHECK-NEXT: store [[WIDE_LOAD3]], ptr [[TMP30]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP12]] ; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -131,34 +131,34 @@ define i32 @multiple_exits_multiple_exit_blocks(ptr %A, ptr %B, i32 %N) #0 { ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP10]] +; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP12:%.*]] = mul i32 [[TMP11]], 8 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[INDEX]], 0 -; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP13:%.*]] = mul i32 [[TMP12]], 4 -; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 0 -; CHECK-NEXT: [[TMP15:%.*]] = mul i32 [[TMP14]], 1 -; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[INDEX]], [[TMP15]] -; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP11]] -; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP16]] -; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP19]], align 4 -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i64 [[TMP21]] -; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP22]], align 4 -; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP11]] -; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP16]] -; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i32 0 -; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP25]], align 4 -; CHECK-NEXT: [[TMP26:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP27:%.*]] = mul i64 [[TMP26]], 4 -; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i64 [[TMP27]] -; CHECK-NEXT: store [[WIDE_LOAD3]], ptr [[TMP28]], align 4 -; CHECK-NEXT: [[TMP29:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP30:%.*]] = mul i32 [[TMP29]], 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP30]] +; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[INDEX]], 0 +; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP15:%.*]] = mul i32 [[TMP14]], 4 +; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], 0 +; CHECK-NEXT: [[TMP17:%.*]] = mul i32 [[TMP16]], 1 +; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[INDEX]], [[TMP17]] +; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP13]] +; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP18]] +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 4 +; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 [[TMP23]] +; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP24]], align 4 +; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP13]] +; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP18]] +; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i32 0 +; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP27]], align 4 +; CHECK-NEXT: [[TMP28:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP29:%.*]] = mul i64 [[TMP28]], 4 +; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP29]] +; CHECK-NEXT: store [[WIDE_LOAD3]], ptr [[TMP30]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP12]] ; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll index 166d771482441732d35e67696a0e8c2dbc56d9b7..1d5c60bd7374efd194f8fa3178277fc8702a6f1e 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll @@ -47,6 +47,8 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr ; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP16]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP49:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP50:%.*]] = mul i64 [[TMP49]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -90,8 +92,6 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr ; CHECK-NEXT: [[TMP47:%.*]] = mul i64 [[TMP46]], 2 ; CHECK-NEXT: [[TMP48:%.*]] = getelementptr i64, ptr [[TMP39]], i64 [[TMP47]] ; CHECK-NEXT: store [[TMP36]], ptr [[TMP48]], align 8 -; CHECK-NEXT: [[TMP49:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP50:%.*]] = mul i64 [[TMP49]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP50]] ; CHECK-NEXT: [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP51]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll index 2ca5fbebed87721c105da5225137f78d9a330cd6..3aac54114c51726a7ad8ee3a386309fe9f2b59a5 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll @@ -10,6 +10,7 @@ target triple = "aarch64-unknown-linux-gnu" ; VPLANS-LABEL: Checking a loop in 'simple_memset' ; VPLANS: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF>=1' { +; VPLANS-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; VPLANS-NEXT: vp<[[TC:%[0-9]+]]> = original trip-count ; VPLANS-EMPTY: ; VPLANS-NEXT: ph: @@ -29,7 +30,7 @@ target triple = "aarch64-unknown-linux-gnu" ; VPLANS-NEXT: vp<[[STEP:%[0-9]+]]> = SCALAR-STEPS vp<[[INDV]]>, ir<1> ; VPLANS-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]> ; VPLANS-NEXT: WIDEN store ir<%gep>, ir<%val>, vp<[[LANEMASK_PHI]]> -; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%[0-9]+]]> = VF * UF + vp<[[INDV]]> +; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%[0-9]+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]> ; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV]]> ; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = active lane mask vp<[[INC]]>, vp<[[NEWTC]]> ; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]> @@ -51,6 +52,8 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -67,8 +70,6 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP10]] ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i32 0 ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP14]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP15:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll index 275a26bae1f2e38e86e5154390b311abb2881714..568046ca29f587d05453fbfea49904880e813856 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll @@ -16,24 +16,24 @@ define void @trip1025_i64(ptr noalias nocapture noundef %dst, ptr noalias nocapt ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 ; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP8:%.*]] = shl nsw [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DST:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP9]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP10]], i32 8, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP11:%.*]] = add nsw [[WIDE_MASKED_LOAD1]], [[TMP8]] -; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP11]], ptr [[TMP10]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP10:%.*]] = shl nsw [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[DST:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP12]], i32 8, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP13:%.*]] = add nsw [[WIDE_MASKED_LOAD1]], [[TMP10]] +; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP13]], ptr [[TMP12]], i32 8, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; CHECK-NEXT: [[TMP14:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP15:%.*]] = extractelement [[TMP14]], i32 0 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-overflow-checks.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-overflow-checks.ll index c0e736399f2932d49861ff3b7036b95a1c413bf6..df107847e3e326bc5aa68d44c364a6ba82d1bb68 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-overflow-checks.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-overflow-checks.ll @@ -15,6 +15,8 @@ define void @cannot_overflow_i32_induction_var(ptr noalias %dst, ptr readonly %s ; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[WIDE_TRIP_COUNT]]) ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: @@ -25,8 +27,6 @@ define void @cannot_overflow_i32_induction_var(ptr noalias %dst, ptr readonly %s ; CHECK-NEXT: [[TMP1:%.*]] = add nsw [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i32 42, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[INDEX]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP1]], ptr [[TMP2]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[WIDE_TRIP_COUNT]]) ; CHECK-NEXT: [[TMP5:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 @@ -76,6 +76,8 @@ define void @can_overflow_i64_induction_var(ptr noalias %dst, ptr readonly %src, ; CHECK: for.body.preheader: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2 ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP1]]) @@ -89,8 +91,6 @@ define void @can_overflow_i64_induction_var(ptr noalias %dst, ptr readonly %src, ; CHECK-NEXT: [[TMP4:%.*]] = add nsw [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i32 42, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[INDEX]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP4]], ptr [[TMP5]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP2]]) ; CHECK-NEXT: [[TMP8:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll index 7e0f18b9efae1b447fd7a8a0710137d865757de5..70e50992b438d92e2a08e76a8f0837f43937d1ae 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll @@ -20,6 +20,8 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -37,8 +39,6 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: [[TMP13:%.*]] = add [[VEC_PHI]], [[WIDE_MASKED_LOAD]] ; CHECK-NEXT: [[TMP14]] = select [[ACTIVE_LANE_MASK]], [[TMP13]], [[VEC_PHI]] -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP17:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -77,6 +77,8 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 { ; CHECK-IN-LOOP-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-IN-LOOP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-IN-LOOP-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-IN-LOOP-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-IN-LOOP-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 ; CHECK-IN-LOOP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-IN-LOOP-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-IN-LOOP-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -95,8 +97,6 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 { ; CHECK-IN-LOOP-NEXT: [[TMP13:%.*]] = select [[ACTIVE_LANE_MASK]], [[WIDE_MASKED_LOAD]], zeroinitializer ; CHECK-IN-LOOP-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP13]]) ; CHECK-IN-LOOP-NEXT: [[TMP15]] = add i32 [[TMP14]], [[VEC_PHI]] -; CHECK-IN-LOOP-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-IN-LOOP-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 ; CHECK-IN-LOOP-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP17]] ; CHECK-IN-LOOP-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-IN-LOOP-NEXT: [[TMP18:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -152,6 +152,8 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -169,8 +171,6 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: [[TMP13:%.*]] = select [[ACTIVE_LANE_MASK]], [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP14]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[TMP13]]) -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP17:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -208,6 +208,8 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 { ; CHECK-IN-LOOP-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-IN-LOOP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-IN-LOOP-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-IN-LOOP-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-IN-LOOP-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-IN-LOOP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-IN-LOOP-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-IN-LOOP-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -225,8 +227,6 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 { ; CHECK-IN-LOOP-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-IN-LOOP-NEXT: [[TMP13:%.*]] = select [[ACTIVE_LANE_MASK]], [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-IN-LOOP-NEXT: [[TMP14]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[TMP13]]) -; CHECK-IN-LOOP-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-IN-LOOP-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-IN-LOOP-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]] ; CHECK-IN-LOOP-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-IN-LOOP-NEXT: [[TMP17:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -281,6 +281,8 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -306,8 +308,6 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 { ; CHECK-NEXT: [[TMP19:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP18]], zeroinitializer ; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP15]], [[TMP17]], [[VEC_PHI]] ; CHECK-NEXT: [[TMP20]] = select [[ACTIVE_LANE_MASK]], [[PREDPHI]], [[VEC_PHI]] -; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP22]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP23:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -353,6 +353,8 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 { ; CHECK-IN-LOOP-NEXT: [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]] ; CHECK-IN-LOOP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-IN-LOOP-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-IN-LOOP-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-IN-LOOP-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; CHECK-IN-LOOP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-IN-LOOP-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-IN-LOOP-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -376,8 +378,6 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 { ; CHECK-IN-LOOP-NEXT: [[TMP17:%.*]] = select [[TMP15]], [[WIDE_MASKED_LOAD1]], zeroinitializer ; CHECK-IN-LOOP-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32( [[TMP17]]) ; CHECK-IN-LOOP-NEXT: [[TMP19]] = xor i32 [[TMP18]], [[VEC_PHI]] -; CHECK-IN-LOOP-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-IN-LOOP-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; CHECK-IN-LOOP-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP21]] ; CHECK-IN-LOOP-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]]) ; CHECK-IN-LOOP-NEXT: [[TMP22:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll index caeb37bce2f18a18bbfa3278de2ab7dedef5102f..4f73d7c1a7735208c24135b9625b93b0244201ce 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll @@ -18,6 +18,8 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP61:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP62:%.*]] = mul i64 [[TMP61]], 16 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 16 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -94,8 +96,6 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP59:%.*]] = mul i64 [[TMP58]], 12 ; CHECK-NEXT: [[TMP60:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP59]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP60]], i32 4, [[ACTIVE_LANE_MASK9]]) -; CHECK-NEXT: [[TMP61:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP62:%.*]] = mul i64 [[TMP61]], 16 ; CHECK-NEXT: [[INDEX_NEXT10]] = add i64 [[INDEX6]], [[TMP62]] ; CHECK-NEXT: [[TMP63:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP64:%.*]] = mul i64 [[TMP63]], 4 @@ -160,6 +160,8 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP83:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP84:%.*]] = mul i64 [[TMP83]], 16 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 16 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -262,8 +264,6 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: [[TMP81:%.*]] = mul i64 [[TMP80]], 12 ; CHECK-NEXT: [[TMP82:%.*]] = getelementptr i32, ptr [[TMP65]], i64 [[TMP81]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP82]], i32 4, [[TMP72]]) -; CHECK-NEXT: [[TMP83:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP84:%.*]] = mul i64 [[TMP83]], 16 ; CHECK-NEXT: [[INDEX_NEXT13]] = add i64 [[INDEX6]], [[TMP84]] ; CHECK-NEXT: [[TMP85:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP86:%.*]] = mul i64 [[TMP85]], 4 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll index 292bdd5c8be85aecd2eda68639a210649d53dbd5..579e8d4ebb3c3adf23437fb265aa1017cea4da59 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll @@ -18,6 +18,8 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -34,8 +36,6 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP10]] ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i32 0 ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP14]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP15:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -145,6 +145,8 @@ define void @simple_memcpy(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -162,8 +164,6 @@ define void @simple_memcpy(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[DST:%.*]], i64 [[TMP10]] ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP13]], i32 0 ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[WIDE_MASKED_LOAD]], ptr [[TMP14]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP17:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -222,6 +222,8 @@ define void @copy_stride4(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP4]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] ; CHECK-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 4 +; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 4 ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; CHECK-NEXT: [[TMP10:%.*]] = sub i64 [[TMP2]], [[TMP9]] @@ -246,8 +248,6 @@ define void @copy_stride4(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[TMP19]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[DST:%.*]], [[VEC_IND]] ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[WIDE_MASKED_GATHER]], [[TMP20]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP22]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP12]]) ; CHECK-NEXT: [[TMP23:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -303,6 +303,8 @@ define void @simple_gather_scatter(ptr noalias %dst, ptr noalias %src, ptr noali ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -321,8 +323,6 @@ define void @simple_gather_scatter(ptr noalias %dst, ptr noalias %src, ptr noali ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[TMP13]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[DST:%.*]], [[WIDE_MASKED_LOAD]] ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[WIDE_MASKED_GATHER]], [[TMP14]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP17:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -382,6 +382,8 @@ define void @uniform_load(ptr noalias %dst, ptr noalias readonly %src, i64 %n) # ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -399,8 +401,6 @@ define void @uniform_load(ptr noalias %dst, ptr noalias readonly %src, i64 %n) # ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[TMP10]] ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i32 0 ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP13]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP15]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP16:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -457,6 +457,8 @@ define void @cond_uniform_load(ptr noalias %dst, ptr noalias readonly %src, ptr ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -483,8 +485,6 @@ define void @cond_uniform_load(ptr noalias %dst, ptr noalias readonly %src, ptr ; CHECK-NEXT: [[TMP18:%.*]] = or [[TMP15]], [[TMP16]] ; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i32 0 ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[PREDPHI]], ptr [[TMP19]], i32 4, [[TMP18]]) -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP21]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP22:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -557,6 +557,8 @@ define void @uniform_store(ptr noalias %dst, ptr noalias readonly %src, i64 %n) ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -574,8 +576,6 @@ define void @uniform_store(ptr noalias %dst, ptr noalias readonly %src, i64 %n) ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0 ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP14]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP15:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -629,6 +629,8 @@ define void @simple_fdiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -648,8 +650,6 @@ define void @simple_fdiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr [[TMP14]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: [[TMP15:%.*]] = fdiv [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD2]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4f32.p0( [[TMP15]], ptr [[TMP14]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 ; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP17]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP18:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -709,6 +709,8 @@ define void @simple_idiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[TMP17]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -729,8 +731,6 @@ define void @simple_idiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[TMP15:%.*]] = select [[ACTIVE_LANE_MASK]], [[WIDE_MASKED_LOAD2]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP16:%.*]] = udiv [[WIDE_MASKED_LOAD]], [[TMP15]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP16]], ptr [[TMP14]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[TMP17]], 4 ; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP18]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP19:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -786,6 +786,8 @@ define void @simple_memset_trip1024(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[VAL:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -795,8 +797,6 @@ define void @simple_memset_trip1024(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP4]] ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i32 0 ; CHECK-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP8]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll index b36a10b46734cd06764e3548a606e4773602f7d6..2cd52bfa87b99e562e16baaec3d2514e44f169b6 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll @@ -23,6 +23,8 @@ define void @vector_reverse_f64(i64 %N, ptr noalias %a, ptr noalias %b) #0{ ; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP30:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP31:%.*]] = shl i64 [[TMP30]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -56,8 +58,6 @@ define void @vector_reverse_f64(i64 %N, ptr noalias %a, ptr noalias %b) #0{ ; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds double, ptr [[TMP19]], i64 [[TMP26]] ; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds double, ptr [[TMP28]], i64 [[TMP27]] ; CHECK-NEXT: store [[TMP18]], ptr [[TMP29]], align 8 -; CHECK-NEXT: [[TMP30:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP31:%.*]] = shl i64 [[TMP30]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP31]] ; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -125,6 +125,8 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 { ; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP33:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP34:%.*]] = shl i64 [[TMP33]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -158,8 +160,6 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 { ; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds i64, ptr [[TMP22]], i64 [[TMP29]] ; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds i64, ptr [[TMP31]], i64 [[TMP30]] ; CHECK-NEXT: store [[TMP21]], ptr [[TMP32]], align 8 -; CHECK-NEXT: [[TMP33:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP34:%.*]] = shl i64 [[TMP33]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP34]] ; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP35]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll index 31cf2c4e2db4cdf1de0d7227d842ea5fb1bd2c47..c3ace983fd9114c969a5bca16d09d995dd446d3b 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll @@ -7,6 +7,8 @@ define void @test_big_little_params(ptr readonly %a, ptr readonly %b, ptr noalia ; CHECK-LABEL: define void @test_big_little_params ; CHECK-SAME: (ptr readonly [[A:%.*]], ptr readonly [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2 ; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025) ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: @@ -19,8 +21,6 @@ define void @test_big_little_params(ptr readonly %a, ptr readonly %b, ptr noalia ; CHECK-NEXT: [[TMP2:%.*]] = call @foo_vector( [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]], [[ACTIVE_LANE_MASK]]) ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP2]], ptr [[TMP3]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; CHECK-NEXT: [[TMP6:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 @@ -52,6 +52,8 @@ define void @test_little_big_params(ptr readonly %a, ptr readonly %b, ptr noalia ; CHECK-LABEL: define void @test_little_big_params ; CHECK-SAME: (ptr readonly [[A:%.*]], ptr readonly [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 1 ; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: @@ -64,8 +66,6 @@ define void @test_little_big_params(ptr readonly %a, ptr readonly %b, ptr noalia ; CHECK-NEXT: [[TMP2:%.*]] = call @bar_vector( [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]], [[ACTIVE_LANE_MASK]]) ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[C]], i64 [[INDEX]] ; CHECK-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP2]], ptr [[TMP3]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; CHECK-NEXT: [[TMP6:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll index 62c778aca51d0a73c47a05bff8591a35accbc59e..140c469327bdcc71407b50a5a13a1473f149af56 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll @@ -9,6 +9,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NOT: LV: Found {{.*}} scalar instruction: %ptr.iv.2.next = getelementptr inbounds i8, ptr %ptr.iv.2, i64 1 ; ; CHECK: VPlan 'Initial VPlan for VF={vscale x 2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%N> = original trip-count ; CHECK-EMPTY: @@ -25,7 +26,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: WIDEN ir<%lv> = load ir<%ptr.iv.2> ; CHECK-NEXT: WIDEN ir<%add> = add ir<%lv>, ir<1> ; CHECK-NEXT: WIDEN store ir<%ptr.iv.2>, ir<%add> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -49,6 +50,8 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias ; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8 ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START_1:%.*]], i64 [[TMP4]] ; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[START_2:%.*]], i64 [[N_VEC]] +; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START_2]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] @@ -75,8 +78,6 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP18]], align 1 ; CHECK-NEXT: [[TMP19:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: store [[TMP19]], ptr [[TMP18]], align 1 -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP21]] ; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP10]] ; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -142,6 +143,8 @@ define void @pointer_induction(ptr noalias %start, i64 %N) { ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]] ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[N_VEC]] +; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] @@ -162,8 +165,6 @@ define void @pointer_induction(ptr noalias %start, i64 %N) { ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP14]], align 1 ; CHECK-NEXT: [[TMP15:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: store [[TMP15]], ptr [[TMP14]], align 1 -; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX2]], [[TMP17]] ; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP8]] ; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll index 9cd2781ae235973d23c16f1b03a2b1cb86ea94d0..c4386c3ce5d1187eb7df82c74406d6d7548da8c5 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll @@ -27,6 +27,8 @@ define void @widen_ptr_phi_unrolled(ptr noalias nocapture %a, ptr noalias nocapt ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] ; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 3 ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 [[TMP3]] +; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP26:%.*]] = shl nuw nsw i64 [[TMP25]], 3 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -61,8 +63,6 @@ define void @widen_ptr_phi_unrolled(ptr noalias nocapture %a, ptr noalias nocapt ; CHECK-NEXT: [[TMP23:%.*]] = shl nuw nsw i64 [[TMP22]], 2 ; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i64 [[TMP23]] ; CHECK-NEXT: store [[TMP20]], ptr [[TMP24]], align 4 -; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP26:%.*]] = shl nuw nsw i64 [[TMP25]], 3 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP26]] ; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -143,6 +143,8 @@ define void @widen_2ptrs_phi_unrolled(ptr noalias nocapture %dst, ptr noalias no ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[TMP3]] ; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2 ; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[TMP4]] +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 3 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -162,8 +164,6 @@ define void @widen_2ptrs_phi_unrolled(ptr noalias nocapture %dst, ptr noalias no ; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP12]], 2 ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[NEXT_GEP5]], i64 [[TMP13]] ; CHECK-NEXT: store [[TMP11]], ptr [[TMP14]], align 4 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 3 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] @@ -234,6 +234,8 @@ define i32 @pointer_iv_mixed(ptr noalias %a, ptr noalias %b, i64 %n) #0 { ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP3]] ; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 3 ; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP4]] +; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP12]], 1 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] @@ -250,8 +252,6 @@ define i32 @pointer_iv_mixed(ptr noalias %a, ptr noalias %b, i64 %n) #0 { ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP10]], align 8 ; CHECK-NEXT: [[TMP11]] = add [[WIDE_LOAD]], [[VEC_PHI]] ; CHECK-NEXT: store [[TMP8]], ptr [[NEXT_GEP]], align 8 -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP12]], 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] ; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP6]] ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -310,6 +310,8 @@ define void @phi_used_in_vector_compare_and_scalar_indvar_update_and_store(ptr % ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 1 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[PTR:%.*]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] @@ -322,8 +324,6 @@ define void @phi_used_in_vector_compare_and_scalar_indvar_update_and_store(ptr % ; CHECK-NEXT: [[TMP4:%.*]] = icmp ne [[TMP3]], zeroinitializer ; CHECK-NEXT: [[TMP5:%.*]] = extractelement [[TMP3]], i64 0 ; CHECK-NEXT: call void @llvm.masked.store.nxv2i16.p0( zeroinitializer, ptr [[TMP5]], i32 2, [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP1]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll index eaabc263913a7db2ce4bb519ede24ad701f05b71..a35c0a4595d942de68dc6bdc77badcb6a4b64eed 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll @@ -10,6 +10,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-LABEL: LV: Checking a loop in 'test_v4_v4m' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: @@ -25,7 +26,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: REPLICATE ir<%call> = call @foo(ir<%load>) ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -36,6 +37,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: @@ -51,7 +53,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask) ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -67,6 +69,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-LABEL: LV: Checking a loop in 'test_v2_v4m' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: @@ -82,7 +85,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask) ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXST:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXST:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -93,6 +96,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: @@ -108,7 +112,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>, ir) (using library function: foo_vector_fixed4_mask) ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -123,6 +127,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-LABEL: LV: Checking a loop in 'test_v2_v4' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: @@ -138,7 +143,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask) ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -149,6 +154,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: @@ -164,7 +170,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask) ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll index 814d3ba53050e31841d37a2ca09c1f3fe9f05687..94f24fea3609c4656939522c21d5fab93db4c436 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll @@ -22,6 +22,8 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; NONE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; NONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP3]] ; NONE-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]] +; NONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; NONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 ; NONE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[VAL:%.*]], i64 0 ; NONE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; NONE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -31,8 +33,6 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; NONE-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP4]] ; NONE-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i32 0 ; NONE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 4 -; NONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; NONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 ; NONE-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP8]] ; NONE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]] ; NONE-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -69,6 +69,8 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]] ; DATA-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] ; DATA-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; DATA-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; DATA-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; DATA-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[VAL:%.*]], i64 0 ; DATA-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; DATA-NEXT: br label [[VECTOR_BODY:%.*]] @@ -79,8 +81,6 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP9]] ; DATA-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[TMP10]], i32 0 ; DATA-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP11]], i32 4, [[ACTIVE_LANE_MASK]]) -; DATA-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; DATA-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; DATA-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP13]] ; DATA-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]] ; DATA-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -119,6 +119,8 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA_NO_LANEMASK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[UMAX]], 1 ; DATA_NO_LANEMASK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0 ; DATA_NO_LANEMASK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer +; DATA_NO_LANEMASK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; DATA_NO_LANEMASK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; DATA_NO_LANEMASK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement poison, i32 [[VAL:%.*]], i64 0 ; DATA_NO_LANEMASK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector [[BROADCAST_SPLATINSERT4]], poison, zeroinitializer ; DATA_NO_LANEMASK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -134,8 +136,6 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA_NO_LANEMASK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP9]] ; DATA_NO_LANEMASK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP13]], i32 0 ; DATA_NO_LANEMASK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT5]], ptr [[TMP14]], i32 4, [[TMP12]]) -; DATA_NO_LANEMASK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; DATA_NO_LANEMASK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; DATA_NO_LANEMASK-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX1]], [[TMP16]] ; DATA_NO_LANEMASK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC]] ; DATA_NO_LANEMASK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -171,6 +171,8 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA_AND_CONTROL-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]] ; DATA_AND_CONTROL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] ; DATA_AND_CONTROL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; DATA_AND_CONTROL-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; DATA_AND_CONTROL-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; DATA_AND_CONTROL-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]]) ; DATA_AND_CONTROL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[VAL:%.*]], i64 0 ; DATA_AND_CONTROL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer @@ -182,8 +184,6 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA_AND_CONTROL-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP9]] ; DATA_AND_CONTROL-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[TMP10]], i32 0 ; DATA_AND_CONTROL-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP11]], i32 4, [[ACTIVE_LANE_MASK]]) -; DATA_AND_CONTROL-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; DATA_AND_CONTROL-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; DATA_AND_CONTROL-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP13]] ; DATA_AND_CONTROL-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]]) ; DATA_AND_CONTROL-NEXT: [[TMP14:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -217,6 +217,8 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() +; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -233,8 +235,6 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP10]] ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i32 0 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]]) -; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP14]] ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP15:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll index f49b3714062d7459286a9a34acd227cdc9e6b8c1..dec3c286345adfc51beb3134a36c15d7ced9c14e 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll @@ -32,18 +32,18 @@ define void @zext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 8 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 8 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP7]], align 1 -; CHECK-NEXT: [[TMP8:%.*]] = zext [[WIDE_LOAD]] to -; CHECK-NEXT: [[TMP9:%.*]] = add [[TMP8]], trunc ( shufflevector ( insertelement ( poison, i32 2, i64 0), poison, zeroinitializer) to ) -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]] -; CHECK-NEXT: store [[TMP9]], ptr [[TMP10]], align 2 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP9]], align 1 +; CHECK-NEXT: [[TMP10:%.*]] = zext [[WIDE_LOAD]] to +; CHECK-NEXT: [[TMP11:%.*]] = add [[TMP10]], trunc ( shufflevector ( insertelement ( poison, i32 2, i64 0), poison, zeroinitializer) to ) +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]] +; CHECK-NEXT: store [[TMP11]], ptr [[TMP12]], align 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -115,18 +115,18 @@ define void @sext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 8 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 8 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP7]], align 1 -; CHECK-NEXT: [[TMP8:%.*]] = sext [[WIDE_LOAD]] to -; CHECK-NEXT: [[TMP9:%.*]] = add [[TMP8]], trunc ( shufflevector ( insertelement ( poison, i32 2, i64 0), poison, zeroinitializer) to ) -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]] -; CHECK-NEXT: store [[TMP9]], ptr [[TMP10]], align 2 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP9]], align 1 +; CHECK-NEXT: [[TMP10:%.*]] = sext [[WIDE_LOAD]] to +; CHECK-NEXT: [[TMP11:%.*]] = add [[TMP10]], trunc ( shufflevector ( insertelement ( poison, i32 2, i64 0), poison, zeroinitializer) to ) +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]] +; CHECK-NEXT: store [[TMP11]], ptr [[TMP12]], align 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll index 51465dbf13e2fe10ae502f83ef81e46e2f4d6752..bda20ae18bc961d0fef179734031d74f716a44b6 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll @@ -9,6 +9,8 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6 ; CHECK-LABEL: define void @test_uniform ; CHECK-SAME: (ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[UNIFORM:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[TMP6]], 1 ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 1 ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP1]]) @@ -22,8 +24,6 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6 ; CHECK-NEXT: [[TMP4:%.*]] = call @foo_uniform( [[WIDE_MASKED_LOAD]], i64 [[UNIFORM]], [[ACTIVE_LANE_MASK]]) ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]] ; CHECK-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP4]], ptr [[TMP5]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[TMP6]], 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP2]]) ; CHECK-NEXT: [[TMP8:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 @@ -53,6 +53,8 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3 ; CHECK-LABEL: define void @test_uniform_smaller_scalar ; CHECK-SAME: (ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[UNIFORM:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[TMP6]], 1 ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 1 ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP1]]) @@ -66,8 +68,6 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3 ; CHECK-NEXT: [[TMP4:%.*]] = call @bar_uniform( [[WIDE_MASKED_LOAD]], i32 [[UNIFORM]], [[ACTIVE_LANE_MASK]]) ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]] ; CHECK-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP4]], ptr [[TMP5]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[TMP6]], 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP2]]) ; CHECK-NEXT: [[TMP8:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-call-linear-args.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-call-linear-args.ll index cd133371f66ce34741fa72c913c4435757cd2978..876d58131bd7a160999ca7a2688a0a205229616a 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/vector-call-linear-args.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-call-linear-args.ll @@ -1,4 +1,4 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "call" --version 2 +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "call.*(foo|bar|baz|quux)" --version 2 ; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -S | FileCheck %s --check-prefixes=NEON ; RUN: opt < %s -mattr=+sve -passes=loop-vectorize -force-vector-interleave=1 -S | FileCheck %s --check-prefixes=SVE_OR_NEON ; RUN: opt < %s -mattr=+sve -passes=loop-vectorize -force-vector-interleave=1 -S -prefer-predicate-over-epilogue=predicate-dont-vectorize | FileCheck %s --check-prefixes=SVE_TF @@ -10,15 +10,18 @@ target triple = "aarch64-unknown-linux-gnu" define void @test_linear8(ptr noalias %a, ptr readnone %b, i64 %n) { ; NEON-LABEL: define void @test_linear8 ; NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) { +; NEON: [[TMP3:%.*]] = call <2 x i64> @vec_foo_linear8_nomask_neon(ptr [[TMP2:%.*]]) ; NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR0:[0-9]+]] ; ; SVE_OR_NEON-LABEL: define void @test_linear8 ; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { -; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR1:[0-9]+]] +; SVE_OR_NEON: [[TMP13:%.*]] = call @vec_foo_linear8_nomask_sve(ptr [[TMP12:%.*]]) +; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR2:[0-9]+]] ; ; SVE_TF-LABEL: define void @test_linear8 ; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { -; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR1:[0-9]+]] +; SVE_TF: [[TMP19:%.*]] = call @vec_foo_linear8_mask_sve(ptr [[TMP18:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR3:[0-9]+]] ; entry: br label %for.body @@ -40,15 +43,17 @@ for.cond.cleanup: define void @test_vector_linear4(ptr noalias %a, ptr readnone %b, ptr readonly %c, i64 %n) { ; NEON-LABEL: define void @test_vector_linear4 ; NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly [[C:%.*]], i64 [[N:%.*]]) { +; NEON: [[TMP5:%.*]] = call <4 x i32> @vec_baz_vector_linear4_nomask_neon(<4 x i32> [[WIDE_LOAD:%.*]], ptr [[TMP4:%.*]]) ; NEON: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr [[GEPB:%.*]]) #[[ATTR1:[0-9]+]] ; ; SVE_OR_NEON-LABEL: define void @test_vector_linear4 ; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_OR_NEON: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr [[GEPB:%.*]]) #[[ATTR2:[0-9]+]] +; SVE_OR_NEON: [[TMP15:%.*]] = call @vec_baz_vector_linear4_nomask_sve( [[WIDE_LOAD:%.*]], ptr [[TMP14:%.*]]) +; SVE_OR_NEON: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr [[GEPB:%.*]]) #[[ATTR3:[0-9]+]] ; ; SVE_TF-LABEL: define void @test_vector_linear4 ; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_TF: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr [[GEPB:%.*]]) #[[ATTR2:[0-9]+]] +; SVE_TF: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr [[GEPB:%.*]]) #[[ATTR4:[0-9]+]] ; entry: br label %for.body @@ -76,11 +81,11 @@ define void @test_linear8_bad_stride(ptr noalias %a, ptr readnone %b, i64 %n) { ; ; SVE_OR_NEON-LABEL: define void @test_linear8_bad_stride ; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR3:[0-9]+]] +; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR4:[0-9]+]] ; ; SVE_TF-LABEL: define void @test_linear8_bad_stride ; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR3:[0-9]+]] +; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR5:[0-9]+]] ; entry: br label %for.body @@ -102,15 +107,17 @@ for.cond.cleanup: define void @test_linear16_wide_stride(ptr noalias %a, ptr readnone %b, i64 %n) { ; NEON-LABEL: define void @test_linear16_wide_stride ; NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) { +; NEON: [[TMP4:%.*]] = call <2 x i64> @vec_foo_linear16_nomask_neon(ptr [[TMP3:%.*]]) ; NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR2]] ; ; SVE_OR_NEON-LABEL: define void @test_linear16_wide_stride ; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR3]] +; SVE_OR_NEON: [[TMP14:%.*]] = call @vec_foo_linear16_nomask_sve(ptr [[TMP13:%.*]]) +; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR4]] ; ; SVE_TF-LABEL: define void @test_linear16_wide_stride ; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR3]] +; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR5]] ; entry: br label %for.body @@ -133,15 +140,18 @@ for.cond.cleanup: define void @test_linear4_linear8(ptr noalias %a, ptr readnone %b, ptr readonly %c, i64 %n) { ; NEON-LABEL: define void @test_linear4_linear8 ; NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly [[C:%.*]], i64 [[N:%.*]]) { +; NEON: [[TMP5:%.*]] = call <4 x i32> @vec_quux_linear4_linear8_nomask_neon(ptr [[TMP3:%.*]], ptr [[TMP4:%.*]]) ; NEON: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr [[GEPB:%.*]]) #[[ATTR3:[0-9]+]] ; ; SVE_OR_NEON-LABEL: define void @test_linear4_linear8 ; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_OR_NEON: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr [[GEPB:%.*]]) #[[ATTR4:[0-9]+]] +; SVE_OR_NEON: [[TMP15:%.*]] = call @vec_quux_linear4_linear8_mask_sve(ptr [[TMP13:%.*]], ptr [[TMP14:%.*]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) +; SVE_OR_NEON: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr [[GEPB:%.*]]) #[[ATTR5:[0-9]+]] ; ; SVE_TF-LABEL: define void @test_linear4_linear8 ; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_TF: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr [[GEPB:%.*]]) #[[ATTR4:[0-9]+]] +; SVE_TF: [[TMP21:%.*]] = call @vec_quux_linear4_linear8_mask_sve(ptr [[TMP19:%.*]], ptr [[TMP20:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; SVE_TF: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr [[GEPB:%.*]]) #[[ATTR6:[0-9]+]] ; entry: br label %for.body @@ -164,15 +174,17 @@ for.cond.cleanup: define void @test_linear3_non_ptr(ptr noalias %a, i64 %n) { ; NEON-LABEL: define void @test_linear3_non_ptr ; NEON-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) { +; NEON: [[TMP3:%.*]] = call <4 x i32> @vec_bar_linear3_nomask_neon(i32 [[TMP2:%.*]]) ; NEON: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]]) #[[ATTR4:[0-9]+]] ; ; SVE_OR_NEON-LABEL: define void @test_linear3_non_ptr ; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_OR_NEON: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]]) #[[ATTR5:[0-9]+]] +; SVE_OR_NEON: [[TMP13:%.*]] = call @vec_bar_linear3_nomask_sve(i32 [[TMP12:%.*]]) +; SVE_OR_NEON: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]]) #[[ATTR6:[0-9]+]] ; ; SVE_TF-LABEL: define void @test_linear3_non_ptr ; SVE_TF-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_TF: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]]) #[[ATTR5:[0-9]+]] +; SVE_TF: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]]) #[[ATTR7:[0-9]+]] ; entry: br label %for.body @@ -195,15 +207,17 @@ for.cond.cleanup: define void @test_linearn5_non_ptr_neg_stride(ptr noalias %a, i64 %n) { ; NEON-LABEL: define void @test_linearn5_non_ptr_neg_stride ; NEON-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) { +; NEON: [[TMP3:%.*]] = call <4 x i32> @vec_bar_linearn5_nomask_neon(i32 [[TMP2:%.*]]) ; NEON: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]]) #[[ATTR5:[0-9]+]] ; ; SVE_OR_NEON-LABEL: define void @test_linearn5_non_ptr_neg_stride ; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_OR_NEON: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]]) #[[ATTR6:[0-9]+]] +; SVE_OR_NEON: [[TMP13:%.*]] = call @vec_bar_linearn5_nomask_sve(i32 [[TMP12:%.*]]) +; SVE_OR_NEON: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]]) #[[ATTR7:[0-9]+]] ; ; SVE_TF-LABEL: define void @test_linearn5_non_ptr_neg_stride ; SVE_TF-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_TF: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]]) #[[ATTR6:[0-9]+]] +; SVE_TF: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]]) #[[ATTR8:[0-9]+]] ; entry: br label %for.body diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll index f82eb9c7822f29b09d994b77b4e7d7633544d715..a3a2223a2528870e81f9c2dee74b1fbce0fd614d 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll @@ -7,7 +7,9 @@ target triple = "arm64-apple-ios" ; CHECK-LABEL: LV: Checking a loop in 'test' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count + ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: @@ -23,7 +25,7 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using library function: __simd_sin_v2f64) ; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]> ; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -34,6 +36,7 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: @@ -50,7 +53,7 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using vector intrinsic) ; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]> ; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll index 7d095206c6062b44626a0b9fc47f675f019b1b50..4a2f9d07ed91c640dbeac87a68265dde87c8990a 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll @@ -16,6 +16,8 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #1 { ; WIDE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; WIDE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; WIDE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; WIDE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; WIDE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; WIDE-NEXT: br label [[VECTOR_BODY:%.*]] ; WIDE: vector.body: ; WIDE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -25,8 +27,6 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #1 { ; WIDE-NEXT: [[TMP6:%.*]] = call @foo_vector( [[TMP5]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; WIDE-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]] ; WIDE-NEXT: store [[TMP6]], ptr [[TMP7]], align 4 -; WIDE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; WIDE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; WIDE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; WIDE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; WIDE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/Hexagon/maximum-vf-crash.ll b/llvm/test/Transforms/LoopVectorize/Hexagon/maximum-vf-crash.ll index 4c7a7d31c5e728581a54a49704990b171304d505..4966ddd299492fa53e362bb96b5d40d2321843fc 100644 --- a/llvm/test/Transforms/LoopVectorize/Hexagon/maximum-vf-crash.ll +++ b/llvm/test/Transforms/LoopVectorize/Hexagon/maximum-vf-crash.ll @@ -1,4 +1,4 @@ -; RUN: opt -march=hexagon -hexagon-autohvx -passes=loop-vectorize -S < %s 2>&1 | FileCheck %s +; RUN: opt -mtriple=hexagon -hexagon-autohvx -passes=loop-vectorize -S < %s 2>&1 | FileCheck %s ; Check that we don't crash. @@ -6,7 +6,6 @@ ; CHECK: vector.body target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048" -target triple = "hexagon" ; Function Attrs: optsize define i32 @f() #0 { diff --git a/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll b/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll index 4ff7ad9ab4390e3c7c65910b7301629417461e54..1ac556a2fda39a19866a6c72f6af129e8991ccc2 100644 --- a/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll +++ b/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll @@ -1,4 +1,4 @@ -; RUN: opt -march=hexagon -passes=loop-vectorize -hexagon-autohvx -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s +; RUN: opt -mtriple=hexagon -passes=loop-vectorize -hexagon-autohvx -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s ; REQUIRES: asserts ; Check that TTI::getMinimumVF works. The calculated MaxVF was based on the @@ -6,7 +6,6 @@ ; CHECK: LV: Overriding calculated MaxVF({{[0-9]+}}) with target's minimum: 64 target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048" -target triple = "hexagon" %s.0 = type { ptr, i32, i32, i32, i32 } diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll b/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll index 4b93ea30cf252d8914dae44ebf15c43c3b9cbc42..db0be4c6281575ca8e845d62f6f3098a97276fe9 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll @@ -22,20 +22,20 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = add [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: store [[TMP9]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -85,22 +85,22 @@ define i64 @vector_add_reduce(ptr noalias nocapture %a) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7]] = add [[VEC_PHI]], [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9]] = add [[VEC_PHI]], [[WIDE_LOAD]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64( [[TMP7]]) +; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64( [[TMP9]]) ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll index a6ab8d1a1f35d7faf44d9ef5755237cc0df0cea9..1f1891ee454885e6237e6e7ece5a42391db26640 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll @@ -19,20 +19,20 @@ define void @vector_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = udiv [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = udiv [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: store [[TMP9]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -123,20 +123,20 @@ define void @vector_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = sdiv [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = sdiv [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: store [[TMP9]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: @@ -227,20 +227,20 @@ define void @vector_urem(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = urem [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = urem [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: store [[TMP9]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: middle.block: @@ -331,20 +331,20 @@ define void @vector_srem(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = srem [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = srem [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: store [[TMP9]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: middle.block: @@ -435,24 +435,24 @@ define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[BROADCAST_SPLAT]], zeroinitializer -; CHECK-NEXT: [[TMP8:%.*]] = select [[TMP7]], [[BROADCAST_SPLAT]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP9:%.*]] = udiv [[WIDE_LOAD]], [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = xor [[TMP7]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP7]], [[TMP9]], [[WIDE_LOAD]] -; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne [[BROADCAST_SPLAT]], zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = select [[TMP9]], [[BROADCAST_SPLAT]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP11:%.*]] = udiv [[WIDE_LOAD]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = xor [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP9]], [[TMP11]], [[WIDE_LOAD]] +; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: middle.block: @@ -569,24 +569,24 @@ define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[BROADCAST_SPLAT]], zeroinitializer -; CHECK-NEXT: [[TMP8:%.*]] = select [[TMP7]], [[BROADCAST_SPLAT]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP9:%.*]] = sdiv [[WIDE_LOAD]], [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = xor [[TMP7]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP7]], [[TMP9]], [[WIDE_LOAD]] -; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne [[BROADCAST_SPLAT]], zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = select [[TMP9]], [[BROADCAST_SPLAT]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP11:%.*]] = sdiv [[WIDE_LOAD]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = xor [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP9]], [[TMP11]], [[WIDE_LOAD]] +; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] ; CHECK: middle.block: @@ -703,21 +703,21 @@ define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 42, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP8:%.*]] = udiv [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 27, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP9:%.*]] = xor [[TMP7]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP7]], [[TMP8]], [[WIDE_LOAD]] -; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 42, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP10:%.*]] = udiv [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 27, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP11:%.*]] = xor [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP9]], [[TMP10]], [[WIDE_LOAD]] +; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] ; CHECK: middle.block: @@ -830,21 +830,21 @@ define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 42, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP8:%.*]] = sdiv [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 27, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP9:%.*]] = xor [[TMP7]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP7]], [[TMP8]], [[WIDE_LOAD]] -; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 42, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP10:%.*]] = sdiv [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 27, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP11:%.*]] = xor [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP9]], [[TMP10]], [[WIDE_LOAD]] +; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] ; CHECK: middle.block: @@ -957,22 +957,22 @@ define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 16 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 16 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 1 -; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i8 -128, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP8:%.*]] = select [[TMP7]], shufflevector ( insertelement ( poison, i8 -1, i64 0), poison, zeroinitializer), shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP9:%.*]] = sdiv [[WIDE_LOAD]], [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = xor [[TMP7]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP7]], [[TMP9]], [[WIDE_LOAD]] -; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP6]], align 1 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 16 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 1 +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i8 -128, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP10:%.*]] = select [[TMP9]], shufflevector ( insertelement ( poison, i8 -1, i64 0), poison, zeroinitializer), shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP11:%.*]] = sdiv [[WIDE_LOAD]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = xor [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP9]], [[TMP11]], [[WIDE_LOAD]] +; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP8]], align 1 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll index 1310ed3618b2c587496a476fc11be165a8fd02c6..57e1dc9051f4d03f5e588c6fe2d2d4bb15960d73 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll @@ -21,23 +21,23 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; OUTLOOP-NEXT: [[TMP3:%.*]] = mul i32 [[TMP2]], 4 ; OUTLOOP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP3]] ; OUTLOOP-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]] +; OUTLOOP-NEXT: [[TMP4:%.*]] = call i32 @llvm.vscale.i32() +; OUTLOOP-NEXT: [[TMP5:%.*]] = mul i32 [[TMP4]], 4 ; OUTLOOP-NEXT: br label [[VECTOR_BODY:%.*]] ; OUTLOOP: vector.body: ; OUTLOOP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; OUTLOOP-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ] -; OUTLOOP-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 0 -; OUTLOOP-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP4]] -; OUTLOOP-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[TMP5]], i32 0 -; OUTLOOP-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 2 -; OUTLOOP-NEXT: [[TMP7:%.*]] = sext [[WIDE_LOAD]] to -; OUTLOOP-NEXT: [[TMP8]] = add [[VEC_PHI]], [[TMP7]] -; OUTLOOP-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() -; OUTLOOP-NEXT: [[TMP10:%.*]] = mul i32 [[TMP9]], 4 -; OUTLOOP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP10]] +; OUTLOOP-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ] +; OUTLOOP-NEXT: [[TMP6:%.*]] = add i32 [[INDEX]], 0 +; OUTLOOP-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP6]] +; OUTLOOP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[TMP7]], i32 0 +; OUTLOOP-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 2 +; OUTLOOP-NEXT: [[TMP9:%.*]] = sext [[WIDE_LOAD]] to +; OUTLOOP-NEXT: [[TMP10]] = add [[VEC_PHI]], [[TMP9]] +; OUTLOOP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]] ; OUTLOOP-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; OUTLOOP-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; OUTLOOP: middle.block: -; OUTLOOP-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP8]]) +; OUTLOOP-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP10]]) ; OUTLOOP-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]] ; OUTLOOP-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; OUTLOOP: scalar.ph: @@ -75,20 +75,20 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; INLOOP-NEXT: [[TMP3:%.*]] = mul i32 [[TMP2]], 8 ; INLOOP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP3]] ; INLOOP-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]] +; INLOOP-NEXT: [[TMP4:%.*]] = call i32 @llvm.vscale.i32() +; INLOOP-NEXT: [[TMP5:%.*]] = mul i32 [[TMP4]], 8 ; INLOOP-NEXT: br label [[VECTOR_BODY:%.*]] ; INLOOP: vector.body: ; INLOOP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; INLOOP-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ] -; INLOOP-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 0 -; INLOOP-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP4]] -; INLOOP-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[TMP5]], i32 0 -; INLOOP-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 2 -; INLOOP-NEXT: [[TMP7:%.*]] = sext [[WIDE_LOAD]] to -; INLOOP-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32( [[TMP7]]) -; INLOOP-NEXT: [[TMP9]] = add i32 [[TMP8]], [[VEC_PHI]] -; INLOOP-NEXT: [[TMP10:%.*]] = call i32 @llvm.vscale.i32() -; INLOOP-NEXT: [[TMP11:%.*]] = mul i32 [[TMP10]], 8 -; INLOOP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP11]] +; INLOOP-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ] +; INLOOP-NEXT: [[TMP6:%.*]] = add i32 [[INDEX]], 0 +; INLOOP-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP6]] +; INLOOP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[TMP7]], i32 0 +; INLOOP-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 2 +; INLOOP-NEXT: [[TMP9:%.*]] = sext [[WIDE_LOAD]] to +; INLOOP-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32( [[TMP9]]) +; INLOOP-NEXT: [[TMP11]] = add i32 [[TMP10]], [[VEC_PHI]] +; INLOOP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]] ; INLOOP-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; INLOOP-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; INLOOP: middle.block: @@ -96,7 +96,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; INLOOP-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; INLOOP: scalar.ph: ; INLOOP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ] -; INLOOP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ] +; INLOOP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ] ; INLOOP-NEXT: br label [[FOR_BODY:%.*]] ; INLOOP: for.body: ; INLOOP-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] @@ -109,7 +109,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; INLOOP-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]] ; INLOOP-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; INLOOP: for.cond.cleanup.loopexit: -; INLOOP-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ] +; INLOOP-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ] ; INLOOP-NEXT: br label [[FOR_COND_CLEANUP]] ; INLOOP: for.cond.cleanup: ; INLOOP-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll index 29ef7364b821286ee2d5a7ba965b88557b5da909..a9cd91b5a8cb565761b9b2d49b471182062e8576 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll @@ -248,6 +248,8 @@ define void @load_store_factor3_i64(ptr %p) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 2 ; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; CHECK-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -276,8 +278,6 @@ define void @load_store_factor3_i64(ptr %p) { ; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call @llvm.masked.gather.nxv2i64.nxv2p0( [[TMP17]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) ; CHECK-NEXT: [[TMP18:%.*]] = add [[WIDE_MASKED_GATHER2]], shufflevector ( insertelement ( poison, i64 3, i64 0), poison, zeroinitializer) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0( [[TMP18]], [[TMP17]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP20]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -353,6 +353,8 @@ define void @load_store_factor8(ptr %p) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP34:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP34]], 2 ; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; CHECK-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -406,8 +408,6 @@ define void @load_store_factor8(ptr %p) { ; CHECK-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = call @llvm.masked.gather.nxv2i64.nxv2p0( [[TMP32]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) ; CHECK-NEXT: [[TMP33:%.*]] = add [[WIDE_MASKED_GATHER7]], shufflevector ( insertelement ( poison, i64 8, i64 0), poison, zeroinitializer) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0( [[TMP33]], [[TMP32]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP34:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP34]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP35]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll b/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll index 072cef66c00f434da09ba0c2b427e93ff41a1bcb..b0aefae41a1fccee65d4d351fd7dcda210c1e7c6 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll @@ -15,6 +15,7 @@ define void @load_store(ptr %p) { ; LMUL1-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() ; LMUL1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]] ; LMUL1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; LMUL1-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() ; LMUL1-NEXT: br label [[VECTOR_BODY:%.*]] ; LMUL1: vector.body: ; LMUL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -24,7 +25,6 @@ define void @load_store(ptr %p) { ; LMUL1-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP4]], align 8 ; LMUL1-NEXT: [[TMP5:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; LMUL1-NEXT: store [[TMP5]], ptr [[TMP4]], align 8 -; LMUL1-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() ; LMUL1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] ; LMUL1-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; LMUL1-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -57,6 +57,8 @@ define void @load_store(ptr %p) { ; LMUL2-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; LMUL2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; LMUL2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; LMUL2-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; LMUL2-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; LMUL2-NEXT: br label [[VECTOR_BODY:%.*]] ; LMUL2: vector.body: ; LMUL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -66,8 +68,6 @@ define void @load_store(ptr %p) { ; LMUL2-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 ; LMUL2-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; LMUL2-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; LMUL2-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; LMUL2-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; LMUL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; LMUL2-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; LMUL2-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -100,6 +100,8 @@ define void @load_store(ptr %p) { ; LMUL4-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; LMUL4-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; LMUL4-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; LMUL4-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; LMUL4-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; LMUL4-NEXT: br label [[VECTOR_BODY:%.*]] ; LMUL4: vector.body: ; LMUL4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -109,8 +111,6 @@ define void @load_store(ptr %p) { ; LMUL4-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 ; LMUL4-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; LMUL4-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; LMUL4-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; LMUL4-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; LMUL4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; LMUL4-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; LMUL4-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -143,6 +143,8 @@ define void @load_store(ptr %p) { ; LMUL8-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 8 ; LMUL8-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; LMUL8-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; LMUL8-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; LMUL8-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 8 ; LMUL8-NEXT: br label [[VECTOR_BODY:%.*]] ; LMUL8: vector.body: ; LMUL8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -152,8 +154,6 @@ define void @load_store(ptr %p) { ; LMUL8-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 ; LMUL8-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; LMUL8-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; LMUL8-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; LMUL8-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 8 ; LMUL8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; LMUL8-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; LMUL8-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll index ace267d72dea06ce20d913ac94f49befe2339112..0c5394cb95a610de3e65dc127cace76d9c262948 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll @@ -16,23 +16,23 @@ define void @trip5_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 5, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 16 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP5]], i64 5) -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr [[TMP7]], i32 1, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP8:%.*]] = shl [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr [[TMP10]], i32 1, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP11:%.*]] = add [[TMP8]], [[WIDE_MASKED_LOAD1]] -; CHECK-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP11]], ptr [[TMP10]], i32 1, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 16 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP7]], i64 5) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr [[TMP9]], i32 1, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP10:%.*]] = shl [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr [[TMP12]], i32 1, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP13:%.*]] = add [[TMP10]], [[WIDE_MASKED_LOAD1]] +; CHECK-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP13]], ptr [[TMP12]], i32 1, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll b/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll index 34b06972dab06b7661e71c2776e3db53052deb33..1307d57f32bc12b18d597f67f3ccd3909ead6e2a 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll @@ -20,14 +20,16 @@ define void @test(ptr noalias nocapture %a, ptr noalias nocapture %b, i32 %v) { ; VLENUNK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; VLENUNK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLENUNK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] -; VLENUNK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv4i64() -; VLENUNK-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer -; VLENUNK-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) -; VLENUNK-NEXT: [[INDUCTION:%.*]] = add zeroinitializer, [[TMP6]] -; VLENUNK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 -; VLENUNK-NEXT: [[TMP9:%.*]] = mul i64 1, [[TMP8]] -; VLENUNK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP9]], i64 0 +; VLENUNK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 +; VLENUNK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv4i64() +; VLENUNK-NEXT: [[TMP7:%.*]] = add [[TMP6]], zeroinitializer +; VLENUNK-NEXT: [[TMP8:%.*]] = mul [[TMP7]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) +; VLENUNK-NEXT: [[INDUCTION:%.*]] = add zeroinitializer, [[TMP8]] +; VLENUNK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 +; VLENUNK-NEXT: [[TMP11:%.*]] = mul i64 1, [[TMP10]] +; VLENUNK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP11]], i64 0 ; VLENUNK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; VLENUNK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[V:%.*]], i64 0 ; VLENUNK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer @@ -35,20 +37,18 @@ define void @test(ptr noalias nocapture %a, ptr noalias nocapture %b, i32 %v) { ; VLENUNK: vector.body: ; VLENUNK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; VLENUNK-NEXT: [[VEC_IND:%.*]] = phi [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; VLENUNK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 0 -; VLENUNK-NEXT: [[TMP11:%.*]] = icmp ult [[VEC_IND]], shufflevector ( insertelement ( poison, i64 512, i64 0), poison, zeroinitializer) -; VLENUNK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[TMP10]] -; VLENUNK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[TMP12]], i32 0 -; VLENUNK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP13]], i32 4, [[TMP11]], poison) -; VLENUNK-NEXT: [[TMP14:%.*]] = xor [[TMP11]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) -; VLENUNK-NEXT: [[PREDPHI:%.*]] = select [[TMP14]], zeroinitializer, [[WIDE_MASKED_LOAD]] -; VLENUNK-NEXT: [[TMP15:%.*]] = add [[PREDPHI]], [[BROADCAST_SPLAT]] -; VLENUNK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP10]] -; VLENUNK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0 -; VLENUNK-NEXT: store [[TMP15]], ptr [[TMP17]], align 4 -; VLENUNK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 -; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] +; VLENUNK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 0 +; VLENUNK-NEXT: [[TMP13:%.*]] = icmp ult [[VEC_IND]], shufflevector ( insertelement ( poison, i64 512, i64 0), poison, zeroinitializer) +; VLENUNK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[TMP12]] +; VLENUNK-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[TMP14]], i32 0 +; VLENUNK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP15]], i32 4, [[TMP13]], poison) +; VLENUNK-NEXT: [[TMP16:%.*]] = xor [[TMP13]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) +; VLENUNK-NEXT: [[PREDPHI:%.*]] = select [[TMP16]], zeroinitializer, [[WIDE_MASKED_LOAD]] +; VLENUNK-NEXT: [[TMP17:%.*]] = add [[PREDPHI]], [[BROADCAST_SPLAT]] +; VLENUNK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP12]] +; VLENUNK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i32 0 +; VLENUNK-NEXT: store [[TMP17]], ptr [[TMP19]], align 4 +; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; VLENUNK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; VLENUNK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLENUNK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll index 72e3bca79a5a53781c4b838a8d8421c0aeb582db..e50d7362365b813aeb0808fa55e675da71749280 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll @@ -39,6 +39,8 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea ; RV32-NEXT: [[N_MOD_VF:%.*]] = urem i64 625, [[TMP4]] ; RV32-NEXT: [[N_VEC:%.*]] = sub i64 625, [[N_MOD_VF]] ; RV32-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 16 +; RV32-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() +; RV32-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 2 ; RV32-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; RV32-NEXT: [[TMP6:%.*]] = add [[TMP5]], zeroinitializer ; RV32-NEXT: [[TMP7:%.*]] = mul [[TMP6]], shufflevector ( insertelement ( poison, i64 16, i64 0), poison, zeroinitializer) @@ -62,8 +64,6 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea ; RV32-NEXT: [[TMP16:%.*]] = fadd [[WIDE_MASKED_GATHER6]], [[TMP15]] ; RV32-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[A]], [[VEC_IND]] ; RV32-NEXT: call void @llvm.masked.scatter.nxv2f64.nxv2p0( [[TMP16]], [[TMP17]], i32 8, [[TMP12]]), !alias.scope !5, !noalias !7 -; RV32-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; RV32-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 2 ; RV32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] ; RV32-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; RV32-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -121,6 +121,8 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea ; RV64-NEXT: [[N_MOD_VF:%.*]] = urem i64 625, [[TMP4]] ; RV64-NEXT: [[N_VEC:%.*]] = sub i64 625, [[N_MOD_VF]] ; RV64-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 16 +; RV64-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() +; RV64-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 2 ; RV64-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; RV64-NEXT: [[TMP6:%.*]] = add [[TMP5]], zeroinitializer ; RV64-NEXT: [[TMP7:%.*]] = mul [[TMP6]], shufflevector ( insertelement ( poison, i64 16, i64 0), poison, zeroinitializer) @@ -144,8 +146,6 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea ; RV64-NEXT: [[TMP16:%.*]] = fadd [[WIDE_MASKED_GATHER6]], [[TMP15]] ; RV64-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[A]], [[VEC_IND]] ; RV64-NEXT: call void @llvm.masked.scatter.nxv2f64.nxv2p0( [[TMP16]], [[TMP17]], i32 8, [[TMP12]]), !alias.scope !5, !noalias !7 -; RV64-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; RV64-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 2 ; RV64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] ; RV64-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; RV64-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll index cad64f5c7e2beef16af0863fce83771437bb5bbd..ca8c85712fc4108423eca458568ba5e3154d6829 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -51,10 +51,11 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1 ; CHECK-NEXT: LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: vp<%1> = original trip-count +; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count ; CHECK: ph: -; CHECK-NEXT: EMIT vp<%1> = EXPAND SCEV (zext i32 %n to i64) +; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64) ; CHECK-NEXT: No successors ; CHECK: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -70,7 +71,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: WIDEN ir<%add9> = add ir<%1>, ir<1> ; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom> ; CHECK-NEXT: WIDEN store ir<%arrayidx3>, ir<%add9> -; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -188,10 +189,11 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1 ; CHECK-NEXT: LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: vp<%1> = original trip-count +; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count ; CHECK: ph: -; CHECK-NEXT: EMIT vp<%1> = EXPAND SCEV (zext i32 %n to i64) +; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64) ; CHECK-NEXT: No successors ; CHECK: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -207,7 +209,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00> ; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom> ; CHECK-NEXT: WIDEN store ir<%arrayidx3>, ir<%conv1> -; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll b/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll index 90c64e33bcef61fb77fe103c9cbb99cde77845fa..77b0ae2f846a3fb3bd2d5a1f1b501fc7cadf4bb7 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll @@ -19,20 +19,20 @@ define void @test(ptr %p) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 32 -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP4]], 200 -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i32 0 -; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP9]], align 32 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 32 +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP6]], 200 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP10]], i32 0 +; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP11]], align 32 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -142,20 +142,20 @@ define void @trivial_due_max_vscale(ptr %p) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 32 -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP4]], 8192 -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i32 0 -; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP9]], align 32 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 32 +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP6]], 8192 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP10]], i32 0 +; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP11]], align 32 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: middle.block: @@ -208,20 +208,20 @@ define void @no_high_lmul_or_interleave(ptr %p) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 32 -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP4]], 1024 -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i32 0 -; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP9]], align 32 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 32 +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP6]], 1024 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP10]], i32 0 +; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP11]], align 32 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll index f7bc4bd35f377611d38c95fd9c0e63b4ddcbd255..c634c2cf7fc675127954bf6cc85387726dcf6f72 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll @@ -24,6 +24,8 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLENUNK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLENUNK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLENUNK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLENUNK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLENUNK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; VLENUNK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLENUNK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -35,8 +37,6 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLENUNK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 ; VLENUNK-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], [[BROADCAST_SPLAT]] ; VLENUNK-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; VLENUNK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; VLENUNK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLENUNK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -69,6 +69,8 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLEN128-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLEN128-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLEN128-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLEN128-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; VLEN128-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] @@ -80,8 +82,6 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLEN128-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 ; VLEN128-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], [[BROADCAST_SPLAT]] ; VLEN128-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; VLEN128-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; VLEN128-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; VLEN128-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLEN128-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -135,6 +135,8 @@ define void @vector_add_i32(ptr noalias nocapture %a, i32 %v, i64 %n) { ; VLENUNK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; VLENUNK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLENUNK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLENUNK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; VLENUNK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[V:%.*]], i64 0 ; VLENUNK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLENUNK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -146,8 +148,6 @@ define void @vector_add_i32(ptr noalias nocapture %a, i32 %v, i64 %n) { ; VLENUNK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; VLENUNK-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], [[BROADCAST_SPLAT]] ; VLENUNK-NEXT: store [[TMP7]], ptr [[TMP6]], align 4 -; VLENUNK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; VLENUNK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLENUNK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -180,6 +180,8 @@ define void @vector_add_i32(ptr noalias nocapture %a, i32 %v, i64 %n) { ; VLEN128-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; VLEN128-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLEN128-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLEN128-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; VLEN128-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[V:%.*]], i64 0 ; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] @@ -191,8 +193,6 @@ define void @vector_add_i32(ptr noalias nocapture %a, i32 %v, i64 %n) { ; VLEN128-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; VLEN128-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], [[BROADCAST_SPLAT]] ; VLEN128-NEXT: store [[TMP7]], ptr [[TMP6]], align 4 -; VLEN128-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; VLEN128-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; VLEN128-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLEN128-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -301,6 +301,8 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; VLENUNK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLENUNK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLENUNK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLENUNK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLENUNK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; VLENUNK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLENUNK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -312,8 +314,6 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; VLENUNK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 ; VLENUNK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_LOAD]] ; VLENUNK-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0( [[BROADCAST_SPLAT]], [[TMP7]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; VLENUNK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; VLENUNK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLENUNK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -346,6 +346,8 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; VLEN128-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLEN128-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLEN128-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLEN128-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; VLEN128-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] @@ -357,8 +359,6 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; VLEN128-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 ; VLEN128-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_LOAD]] ; VLEN128-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0( [[BROADCAST_SPLAT]], [[TMP7]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; VLEN128-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; VLEN128-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; VLEN128-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLEN128-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -409,6 +409,8 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 ; VLENUNK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLENUNK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLENUNK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLENUNK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; VLENUNK-NEXT: br label [[VECTOR_BODY:%.*]] ; VLENUNK: vector.body: ; VLENUNK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -420,8 +422,6 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 ; VLENUNK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_LOAD]] ; VLENUNK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv2i64.nxv2p0( [[TMP7]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) ; VLENUNK-NEXT: [[TMP8]] = add [[VEC_PHI]], [[WIDE_MASKED_GATHER]] -; VLENUNK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] ; VLENUNK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLENUNK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -459,6 +459,8 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 ; VLEN128-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLEN128-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLEN128-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLEN128-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; VLEN128-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] ; VLEN128: vector.body: ; VLEN128-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -470,8 +472,6 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 ; VLEN128-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_LOAD]] ; VLEN128-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv2i64.nxv2p0( [[TMP7]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) ; VLEN128-NEXT: [[TMP8]] = add [[VEC_PHI]], [[WIDE_MASKED_GATHER]] -; VLEN128-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; VLEN128-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] ; VLEN128-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLEN128-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -529,6 +529,8 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLENUNK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLENUNK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLENUNK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLENUNK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLENUNK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; VLENUNK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLENUNK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -538,8 +540,6 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLENUNK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; VLENUNK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 ; VLENUNK-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8 -; VLENUNK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; VLENUNK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLENUNK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -570,6 +570,8 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLEN128-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLEN128-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLEN128-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLEN128-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; VLEN128-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] @@ -579,8 +581,6 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLEN128-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; VLEN128-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 ; VLEN128-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8 -; VLEN128-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; VLEN128-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; VLEN128-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLEN128-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -627,6 +627,8 @@ define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) { ; VLENUNK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLENUNK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLENUNK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLENUNK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLENUNK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[V:%.*]], i64 0 ; VLENUNK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLENUNK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -636,8 +638,6 @@ define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) { ; VLENUNK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; VLENUNK-NEXT: [[TMP6:%.*]] = getelementptr inbounds ptr, ptr [[TMP5]], i32 0 ; VLENUNK-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8 -; VLENUNK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; VLENUNK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLENUNK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -668,6 +668,8 @@ define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) { ; VLEN128-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLEN128-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLEN128-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLEN128-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; VLEN128-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[V:%.*]], i64 0 ; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] @@ -677,8 +679,6 @@ define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) { ; VLEN128-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; VLEN128-NEXT: [[TMP6:%.*]] = getelementptr inbounds ptr, ptr [[TMP5]], i32 0 ; VLEN128-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8 -; VLEN128-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; VLEN128-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; VLEN128-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLEN128-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll index 0a8c7cfda9efbd809a7ac6ec76307264988b2453..affc6fc22c1baef83a283b6789e1de50f7c47864 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll @@ -19,21 +19,21 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025) -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP8:%.*]] = add [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP8]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP7]], i64 1025) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP10:%.*]] = add [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP10]], ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -85,21 +85,21 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025) -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_MASKED_LOAD]] -; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0( [[BROADCAST_SPLAT]], [[TMP8]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP7]], i64 1025) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_MASKED_LOAD]] +; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0( [[BROADCAST_SPLAT]], [[TMP10]], i32 8, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: @@ -149,26 +149,26 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025) -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_MASKED_LOAD]] -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv2i64.nxv2p0( [[TMP8]], i32 8, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP9]] = add [[VEC_PHI]], [[WIDE_MASKED_GATHER]] -; CHECK-NEXT: [[TMP10:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP9]], [[VEC_PHI]] -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP7]], i64 1025) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_MASKED_LOAD]] +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv2i64.nxv2p0( [[TMP10]], i32 8, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP11]] = add [[VEC_PHI]], [[WIDE_MASKED_GATHER]] +; CHECK-NEXT: [[TMP12:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP11]], [[VEC_PHI]] +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64( [[TMP10]]) +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64( [[TMP12]]) ; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] @@ -221,19 +221,19 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025) -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 -; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP7]], i64 1025) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: middle.block: @@ -279,20 +279,20 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025) +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP7]], i64 1025) ; CHECK-NEXT: store i64 [[V]], ptr [[B:%.*]], align 8 -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 -; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: middle.block: @@ -373,21 +373,21 @@ define void @vector_add_trip1024(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 1024, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1024) -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP8:%.*]] = add [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP8]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP7]], i64 1024) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP10:%.*]] = add [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP10]], ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll index c7bb4ca4b6c6c72a38d18dae7cc589e75ecd0c21..34a7987bb40abeb47bf953b52d8d5829d54ef937 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll @@ -38,6 +38,8 @@ define i32 @select_icmp(i32 %x, i32 %y, ptr nocapture readonly %c, i64 %n) #0 { ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[X:%.*]], i64 0 ; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[Y:%.*]], i64 0 @@ -52,8 +54,6 @@ define i32 @select_icmp(i32 %x, i32 %y, ptr nocapture readonly %c, i64 %n) #0 { ; SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; SCALABLE-NEXT: [[TMP8:%.*]] = icmp slt [[WIDE_LOAD]], [[BROADCAST_SPLAT]] ; SCALABLE-NEXT: [[TMP9]] = select [[TMP8]], [[VEC_PHI]], [[BROADCAST_SPLAT2]] -; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; SCALABLE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -113,6 +113,8 @@ define i32 @select_fcmp(float %x, i32 %y, ptr nocapture readonly %c, i64 %n) #0 ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, float [[X:%.*]], i64 0 ; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[Y:%.*]], i64 0 @@ -127,8 +129,6 @@ define i32 @select_fcmp(float %x, i32 %y, ptr nocapture readonly %c, i64 %n) #0 ; SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; SCALABLE-NEXT: [[TMP8:%.*]] = fcmp fast olt [[WIDE_LOAD]], [[BROADCAST_SPLAT]] ; SCALABLE-NEXT: [[TMP9]] = select [[TMP8]], [[VEC_PHI]], [[BROADCAST_SPLAT2]] -; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; SCALABLE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -184,6 +184,8 @@ define i32 @select_const_i32_from_icmp(ptr nocapture readonly %v, i64 %n) #0 { ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; SCALABLE: vector.body: ; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -194,8 +196,6 @@ define i32 @select_const_i32_from_icmp(ptr nocapture readonly %v, i64 %n) #0 { ; SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; SCALABLE-NEXT: [[TMP8:%.*]] = icmp eq [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 3, i64 0), poison, zeroinitializer) ; SCALABLE-NEXT: [[TMP9]] = select [[TMP8]], [[VEC_PHI]], shufflevector ( insertelement ( poison, i32 7, i64 0), poison, zeroinitializer) -; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; SCALABLE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -257,6 +257,8 @@ define i32 @select_i32_from_icmp(ptr nocapture readonly %v, i32 %a, i32 %b, i64 ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[MINMAX_IDENT_SPLATINSERT:%.*]] = insertelement poison, i32 [[A:%.*]], i64 0 ; SCALABLE-NEXT: [[MINMAX_IDENT_SPLAT:%.*]] = shufflevector [[MINMAX_IDENT_SPLATINSERT]], poison, zeroinitializer ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[B:%.*]], i64 0 @@ -271,8 +273,6 @@ define i32 @select_i32_from_icmp(ptr nocapture readonly %v, i32 %a, i32 %b, i64 ; SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; SCALABLE-NEXT: [[TMP8:%.*]] = icmp eq [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 3, i64 0), poison, zeroinitializer) ; SCALABLE-NEXT: [[TMP9]] = select [[TMP8]], [[VEC_PHI]], [[BROADCAST_SPLAT]] -; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; SCALABLE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -330,6 +330,8 @@ define i32 @select_const_i32_from_fcmp(ptr nocapture readonly %v, i64 %n) #0 { ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; SCALABLE: vector.body: ; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -340,8 +342,6 @@ define i32 @select_const_i32_from_fcmp(ptr nocapture readonly %v, i64 %n) #0 { ; SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; SCALABLE-NEXT: [[TMP8:%.*]] = fcmp fast ueq [[WIDE_LOAD]], shufflevector ( insertelement ( poison, float 3.000000e+00, i64 0), poison, zeroinitializer) ; SCALABLE-NEXT: [[TMP9]] = select [[TMP8]], [[VEC_PHI]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) -; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; SCALABLE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -428,6 +428,8 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1 ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; SCALABLE: vector.body: ; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -444,8 +446,6 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1 ; SCALABLE-NEXT: [[TMP13:%.*]] = select [[TMP12]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer), [[VEC_PHI]] ; SCALABLE-NEXT: [[TMP14:%.*]] = xor [[TMP8]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; SCALABLE-NEXT: [[PREDPHI]] = select [[TMP8]], [[TMP13]], [[VEC_PHI]] -; SCALABLE-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; SCALABLE-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll b/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll index 38ef0537e43183b505136df39cb9d49816171ae9..331a8d2eb66e297a1310fd808dc5642a81efe103 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll @@ -14,6 +14,8 @@ define void @small_trip_count_min_vlen_128(ptr nocapture %a) nounwind vscale_ran ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 4, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP10:%.*]] = mul i32 [[TMP9]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -24,8 +26,6 @@ define void @small_trip_count_min_vlen_128(ptr nocapture %a) nounwind vscale_ran ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP7]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: [[TMP8:%.*]] = add nsw [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP8]], ptr [[TMP7]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP10:%.*]] = mul i32 [[TMP9]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP10]] ; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -75,6 +75,8 @@ define void @small_trip_count_min_vlen_32(ptr nocapture %a) nounwind vscale_rang ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 4, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP10:%.*]] = mul i32 [[TMP9]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -85,8 +87,6 @@ define void @small_trip_count_min_vlen_32(ptr nocapture %a) nounwind vscale_rang ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP7]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: [[TMP8:%.*]] = add nsw [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP8]], ptr [[TMP7]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP10:%.*]] = mul i32 [[TMP9]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP10]] ; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll index 65a2cb51b88ffe5db5e8ac0d0a0a1c1917f8ad7c..12fdf2149daf4743617893ebf0cb05397a8e7b07 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll @@ -17,6 +17,8 @@ define void @single_constant_stride_int_scaled(ptr %p) { ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[TMP5]] +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP7:%.*]] = add [[TMP6]], zeroinitializer ; CHECK-NEXT: [[TMP8:%.*]] = mul [[TMP7]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -35,8 +37,6 @@ define void @single_constant_stride_int_scaled(ptr %p) { ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[TMP13]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) ; CHECK-NEXT: [[TMP14:%.*]] = add [[WIDE_MASKED_GATHER]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[TMP14]], [[TMP13]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -90,6 +90,8 @@ define void @single_constant_stride_int_iv(ptr %p) { ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] ; CHECK-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 64 +; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 64, i64 0), poison, zeroinitializer) @@ -107,8 +109,6 @@ define void @single_constant_stride_int_iv(ptr %p) { ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[TMP10]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) ; CHECK-NEXT: [[TMP11:%.*]] = add [[WIDE_MASKED_GATHER]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[TMP11]], [[TMP10]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -236,6 +236,8 @@ define void @single_stride_int_scaled(ptr %p, i64 %stride) { ; NOSTRIDED-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 4 ; NOSTRIDED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]] ; NOSTRIDED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; NOSTRIDED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 ; NOSTRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; NOSTRIDED: vector.body: ; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -245,8 +247,6 @@ define void @single_stride_int_scaled(ptr %p, i64 %stride) { ; NOSTRIDED-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP7]], align 4 ; NOSTRIDED-NEXT: [[TMP8:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; NOSTRIDED-NEXT: store [[TMP8]], ptr [[TMP7]], align 4 -; NOSTRIDED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 ; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] ; NOSTRIDED-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; NOSTRIDED-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -320,6 +320,8 @@ define void @single_stride_int_iv(ptr %p, i64 %stride) { ; NOSTRIDED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]] ; NOSTRIDED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] ; NOSTRIDED-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], [[STRIDE]] +; NOSTRIDED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 ; NOSTRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; NOSTRIDED: vector.body: ; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -329,8 +331,6 @@ define void @single_stride_int_iv(ptr %p, i64 %stride) { ; NOSTRIDED-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP7]], align 4 ; NOSTRIDED-NEXT: [[TMP8:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; NOSTRIDED-NEXT: store [[TMP8]], ptr [[TMP7]], align 4 -; NOSTRIDED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 ; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] ; NOSTRIDED-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; NOSTRIDED-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -452,6 +452,8 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) { ; NOSTRIDED-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 ; NOSTRIDED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP8]] ; NOSTRIDED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; NOSTRIDED-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; NOSTRIDED-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; NOSTRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; NOSTRIDED: vector.body: ; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -463,8 +465,6 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) { ; NOSTRIDED-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[P2]], i64 [[TMP9]] ; NOSTRIDED-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP13]], i32 0 ; NOSTRIDED-NEXT: store [[TMP12]], ptr [[TMP14]], align 4 -; NOSTRIDED-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; NOSTRIDED-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; NOSTRIDED-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; NOSTRIDED-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -518,6 +518,8 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) { ; STRIDED-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; STRIDED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP9]] ; STRIDED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; STRIDED-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; STRIDED-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; STRIDED-NEXT: [[TMP10:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; STRIDED-NEXT: [[TMP11:%.*]] = add [[TMP10]], zeroinitializer ; STRIDED-NEXT: [[TMP12:%.*]] = mul [[TMP11]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -539,8 +541,6 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) { ; STRIDED-NEXT: [[TMP18:%.*]] = add [[WIDE_MASKED_GATHER]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; STRIDED-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[P2]], [[TMP16]] ; STRIDED-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[TMP18]], [[TMP19]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)), !alias.scope !11, !noalias !8 -; STRIDED-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; STRIDED-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP21]] ; STRIDED-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; STRIDED-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -601,6 +601,8 @@ define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) { ; NOSTRIDED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]] ; NOSTRIDED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] ; NOSTRIDED-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], [[STRIDE]] +; NOSTRIDED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 ; NOSTRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; NOSTRIDED: vector.body: ; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -610,8 +612,6 @@ define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) { ; NOSTRIDED-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP7]], align 4 ; NOSTRIDED-NEXT: [[TMP8:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; NOSTRIDED-NEXT: store [[TMP8]], ptr [[TMP7]], align 4 -; NOSTRIDED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 ; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] ; NOSTRIDED-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; NOSTRIDED-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] @@ -729,6 +729,8 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) { ; STRIDED-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP10]] ; STRIDED-NEXT: [[TMP11:%.*]] = mul i64 [[N_VEC]], [[STRIDE]] ; STRIDED-NEXT: [[IND_END7:%.*]] = getelementptr i8, ptr [[P2]], i64 [[TMP11]] +; STRIDED-NEXT: [[TMP29:%.*]] = call i64 @llvm.vscale.i64() +; STRIDED-NEXT: [[TMP30:%.*]] = mul i64 [[TMP29]], 4 ; STRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; STRIDED: vector.body: ; STRIDED-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[P]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] @@ -761,8 +763,6 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) { ; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[TMP19]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison), !alias.scope !15 ; STRIDED-NEXT: [[TMP28:%.*]] = add [[WIDE_MASKED_GATHER]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; STRIDED-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[TMP28]], [[TMP27]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)), !alias.scope !18, !noalias !15 -; STRIDED-NEXT: [[TMP29:%.*]] = call i64 @llvm.vscale.i64() -; STRIDED-NEXT: [[TMP30:%.*]] = mul i64 [[TMP29]], 4 ; STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP30]] ; STRIDED-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP15]] ; STRIDED-NEXT: [[PTR_IND12]] = getelementptr i8, ptr [[POINTER_PHI11]], i64 [[TMP23]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll index 89c3219c4b6d0b17e235cf644dcb5ece3bc4fd55..a12dfbaec195b11dcbe2ad16b831ad0381c7a385 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll @@ -21,6 +21,8 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6 ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; SCALABLE: vector.body: ; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -31,8 +33,6 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6 ; SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 ; SCALABLE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP7]], align 8 -; SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; SCALABLE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -102,6 +102,8 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6 ; TF-SCALABLE-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TF-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TF-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TF-SCALABLE-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; TF-SCALABLE-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; TF-SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; TF-SCALABLE: vector.body: ; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -113,8 +115,6 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6 ; TF-SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] ; TF-SCALABLE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 ; TF-SCALABLE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP8]], i32 8, [[ACTIVE_LANE_MASK]]) -; TF-SCALABLE-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; TF-SCALABLE-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; TF-SCALABLE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]] ; TF-SCALABLE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TF-SCALABLE-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -196,6 +196,8 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; SCALABLE: vector.body: ; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -206,8 +208,6 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap ; SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 ; SCALABLE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP7]], align 8 -; SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; SCALABLE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -326,6 +326,8 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 ; SCALABLE-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; SCALABLE-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer ; SCALABLE-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -349,8 +351,6 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca ; SCALABLE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP10]] ; SCALABLE-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i32 0 ; SCALABLE-NEXT: store [[PREDPHI]], ptr [[TMP14]], align 8 -; SCALABLE-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; SCALABLE-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; SCALABLE-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -443,6 +443,8 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca ; TF-SCALABLE-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TF-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TF-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TF-SCALABLE-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() +; TF-SCALABLE-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 2 ; TF-SCALABLE-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; TF-SCALABLE-NEXT: [[TMP6:%.*]] = add [[TMP5]], zeroinitializer ; TF-SCALABLE-NEXT: [[TMP7:%.*]] = mul [[TMP6]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -470,8 +472,6 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca ; TF-SCALABLE-NEXT: [[TMP17:%.*]] = or [[TMP13]], [[TMP15]] ; TF-SCALABLE-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i32 0 ; TF-SCALABLE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[PREDPHI]], ptr [[TMP18]], i32 8, [[TMP17]]) -; TF-SCALABLE-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() -; TF-SCALABLE-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 2 ; TF-SCALABLE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP20]] ; TF-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; TF-SCALABLE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -581,6 +581,8 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; SCALABLE: vector.body: ; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -591,8 +593,6 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt ; SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 ; SCALABLE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP7]], align 8 -; SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; SCALABLE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -662,6 +662,8 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt ; TF-SCALABLE-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TF-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TF-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TF-SCALABLE-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; TF-SCALABLE-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; TF-SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; TF-SCALABLE: vector.body: ; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -673,8 +675,6 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt ; TF-SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] ; TF-SCALABLE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 ; TF-SCALABLE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP8]], i32 8, [[ACTIVE_LANE_MASK]]) -; TF-SCALABLE-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; TF-SCALABLE-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; TF-SCALABLE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]] ; TF-SCALABLE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TF-SCALABLE-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -756,6 +756,8 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -766,8 +768,6 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; SCALABLE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 ; SCALABLE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8 -; SCALABLE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; SCALABLE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -837,6 +837,8 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; TF-SCALABLE-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TF-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TF-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TF-SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; TF-SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TF-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; TF-SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -848,8 +850,6 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; TF-SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] ; TF-SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 ; TF-SCALABLE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; TF-SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; TF-SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TF-SCALABLE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] ; TF-SCALABLE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TF-SCALABLE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -931,6 +931,8 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 2 ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -952,8 +954,6 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias ; SCALABLE-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP8]] ; SCALABLE-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP14]], i32 0 ; SCALABLE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP15]], align 8 -; SCALABLE-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]] ; SCALABLE-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -1029,6 +1029,8 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias ; TF-SCALABLE-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TF-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TF-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TF-SCALABLE-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; TF-SCALABLE-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 2 ; TF-SCALABLE-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; TF-SCALABLE-NEXT: [[TMP6:%.*]] = add [[TMP5]], zeroinitializer ; TF-SCALABLE-NEXT: [[TMP7:%.*]] = mul [[TMP6]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -1052,8 +1054,6 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias ; TF-SCALABLE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP11]] ; TF-SCALABLE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0 ; TF-SCALABLE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT2]], ptr [[TMP13]], i32 8, [[ACTIVE_LANE_MASK]]) -; TF-SCALABLE-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; TF-SCALABLE-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 2 ; TF-SCALABLE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP15]] ; TF-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; TF-SCALABLE-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1162,6 +1162,8 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 2 ; SCALABLE-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; SCALABLE-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer ; SCALABLE-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -1185,8 +1187,6 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc ; SCALABLE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP10]] ; SCALABLE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0 ; SCALABLE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP13]], align 8 -; SCALABLE-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] ; SCALABLE-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; SCALABLE-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1275,6 +1275,8 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc ; TF-SCALABLE-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TF-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TF-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TF-SCALABLE-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() +; TF-SCALABLE-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 2 ; TF-SCALABLE-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; TF-SCALABLE-NEXT: [[TMP6:%.*]] = add [[TMP5]], zeroinitializer ; TF-SCALABLE-NEXT: [[TMP7:%.*]] = mul [[TMP6]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -1303,8 +1305,6 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc ; TF-SCALABLE-NEXT: [[TMP17:%.*]] = or [[TMP13]], [[TMP16]] ; TF-SCALABLE-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP14]], i32 0 ; TF-SCALABLE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP18]], i32 8, [[TMP17]]) -; TF-SCALABLE-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() -; TF-SCALABLE-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 2 ; TF-SCALABLE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP20]] ; TF-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; TF-SCALABLE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1412,6 +1412,8 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -1422,8 +1424,6 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap ; SCALABLE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 ; SCALABLE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8 -; SCALABLE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; SCALABLE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] @@ -1493,6 +1493,8 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap ; TF-SCALABLE-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TF-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TF-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TF-SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; TF-SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TF-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; TF-SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -1504,8 +1506,6 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap ; TF-SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] ; TF-SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 ; TF-SCALABLE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; TF-SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; TF-SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TF-SCALABLE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] ; TF-SCALABLE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TF-SCALABLE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll index 8ff542a36732d09aa6dfdcdf2b7880bddb418457..d93cf2b14f8ae559c7faabdaaed50d685f88f45e 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll @@ -5,6 +5,7 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) { ; CHECK-LABEL: 'test_chained_first_order_recurrences_1' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: @@ -23,7 +24,7 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) { ; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE:%.+]]> = first-order splice ir<%for.2>, vp<[[FOR1_SPLICE]]> ; CHECK-NEXT: WIDEN ir<%add> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]> ; CHECK-NEXT: WIDEN store ir<%gep.ptr>, ir<%add> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -55,6 +56,7 @@ exit: define void @test_chained_first_order_recurrences_3(ptr %ptr) { ; CHECK-LABEL: 'test_chained_first_order_recurrences_3' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: @@ -76,7 +78,7 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) { ; CHECK-NEXT: WIDEN ir<%add.1> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]> ; CHECK-NEXT: WIDEN ir<%add.2> = add ir<%add.1>, vp<[[FOR3_SPLICE]]> ; CHECK-NEXT: WIDEN store ir<%gep.ptr>, ir<%add.2> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll index 3198f19a8f9a9a49a94a6ba90dca2ce17dd1f3c6..060d28f030f72e1b6afe30064021122efba88239 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -9,6 +9,7 @@ target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize { ; CHECK-LABEL: sink_replicate_region_1 ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<20001> = original trip-count @@ -65,7 +66,7 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: Successor(s): loop.2 ; CHECK-EMPTY: ; CHECK-NEXT: loop.2: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -99,6 +100,7 @@ exit: define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-LABEL: sink_replicate_region_2 ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<20001> = original trip-count @@ -136,7 +138,7 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: Successor(s): loop.1 ; CHECK-EMPTY: ; CHECK-NEXT: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -168,6 +170,7 @@ exit: define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-LABEL: sink_replicate_region_3_reduction ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<20001> = original trip-count @@ -205,7 +208,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<%recur.next> ; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add> ; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]>, ir<%and.red.next>, ir<%and.red> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -242,6 +245,7 @@ exit: define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr %ptr, ptr noalias %dst) optsize { ; CHECK-LABEL: sink_replicate_region_4_requires_split_at_end_of_block ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<20001> = original trip-count @@ -302,7 +306,7 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: Successor(s): loop.3 ; CHECK-EMPTY: ; CHECK: loop.3: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -340,6 +344,7 @@ exit: define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias %dst.2, i32 %x, i8 %y) optsize { ; CHECK-LABEL: sink_replicate_region_after_replicate_region ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -384,7 +389,7 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias ; CHECK-NEXT: Successor(s): loop.3 ; CHECK-EMPTY: ; CHECK-NEXT: loop.3: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -418,6 +423,7 @@ exit: ; preds = %loop define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias %dst) { ; CHECK-LABEL: need_new_block_after_sinking_pr56146 ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<3> = original trip-count @@ -455,7 +461,7 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias ; CHECK-NEXT: Successor(s): loop.1 ; CHECK-EMPTY: ; CHECK-NEXT: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll index 6dac3531277dc895800aab817eaa7142ab0ebf81..b4ab6f7e8ceb70cb39aea6b0c066d47116434f14 100644 --- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll +++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll @@ -37,6 +37,7 @@ for.end: ; Check for crash exposed by D76992. ; CHECK-LABEL: 'test' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<14> = original trip-count @@ -70,7 +71,7 @@ for.end: ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successor ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll index 028e0b7b39cbaddb95707b78054f68cbf5316a3f..7b9d7f7986396b7397cff1e4518ef24185ca82a5 100644 --- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll +++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll @@ -5,6 +5,7 @@ ; DBG-LABEL: 'test_scalarize_call' ; DBG: VPlan 'Initial VPlan for VF={1},UF>=1' { +; DBG-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; DBG-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; DBG-NEXT: vp<[[TC:%.+]]> = original trip-count ; DBG-EMPTY: @@ -23,7 +24,7 @@ ; DBG-NEXT: CLONE ir<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>) ; DBG-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%dst>, vp<[[IV_STEPS]]> ; DBG-NEXT: CLONE store ir<%min>, ir<%arrayidx> -; DBG-NEXT: EMIT vp<[[INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; DBG-NEXT: EMIT vp<[[INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; DBG-NEXT: EMIT branch-on-count vp<[[INC]]>, vp<[[VEC_TC]]> ; DBG-NEXT: No successors ; DBG-NEXT: } @@ -67,7 +68,8 @@ declare i32 @llvm.smin.i32(i32, i32) ; DBG-LABEL: 'test_scalarize_with_branch_cond' -; DBG: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count +; DBG: Live-in vp<[[VFxUF:%.+]]> = VF * UF +; DBG-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; DBG-NEXT: Live-in ir<1000> = original trip-count ; DBG-EMPTY: ; DBG-NEXT: vector.ph: @@ -100,7 +102,7 @@ declare i32 @llvm.smin.i32(i32, i32) ; DBG-NEXT: Successor(s): cond.false.1 ; DBG-EMPTY: ; DBG-NEXT: cond.false.1: -; DBG-NEXT: EMIT vp<[[CAN_IV_INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; DBG-NEXT: EMIT vp<[[CAN_IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; DBG-NEXT: EMIT branch-on-count vp<[[CAN_IV_INC]]>, vp<[[VEC_TC]]> ; DBG-NEXT: No successors ; DBG-NEXT: } @@ -173,6 +175,7 @@ exit: ; DBG-LABEL: 'first_order_recurrence_using_induction' ; DBG: VPlan 'Initial VPlan for VF={1},UF>=1' { +; DBG-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; DBG-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; DBG-NEXT: vp<[[TC:%.+]]> = original trip-count ; DBG-EMPTY: @@ -191,7 +194,7 @@ exit: ; DBG-NEXT: vp<[[SCALAR_STEPS]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> ; DBG-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for>, vp<[[SCALAR_STEPS]]> ; DBG-NEXT: CLONE store vp<[[SPLICE]]>, ir<%dst> -; DBG-NEXT: EMIT vp<[[IV_INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; DBG-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; DBG-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VTC]]> ; DBG-NEXT: No successors ; DBG-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll index c7d7ac3cb9ede34c2680fe060f6b0d1ac15e3db1..9023916d6e1cd28d4e37fe1b4c3d8e88e6673a78 100644 --- a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll +++ b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll @@ -24,6 +24,8 @@ define void @foo() { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 ; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -55,8 +57,6 @@ define void @foo() { ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4f32.nxv4p0( [[VEC_PHI5]], [[TMP10]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: [[TMP16:%.*]] = add nuw nsw [[VEC_IND]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq [[TMP16]], shufflevector ( insertelement ( poison, i64 1024, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll index 69fbb6519f6a2bc0982cb2a63dce6b00fc107f18..3ba57821bc31b0df43e9b6dd627832beb47c8526 100644 --- a/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll +++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll @@ -1,89 +1,91 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 4 ; RUN: opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-interleave=1 -force-vector-width=4 -prefer-inloop-reductions -S | FileCheck %s define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %cond, i64 %N){ -; CHECK-LABEL: @cond_fadd( +; CHECK-LABEL: define float @cond_fadd( +; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[COND:%.*]], i64 [[N:%.*]]) { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[PRED_LOAD_CONTINUE6]] ] -; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[COND:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP26:%.*]], [[PRED_LOAD_CONTINUE6]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[COND]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = fcmp une <4 x float> [[WIDE_LOAD]], -; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0 -; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = fcmp une <4 x float> [[WIDE_LOAD]], +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0 +; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] ; CHECK: pred.load.if: -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[TMP5:%.*]] = load float, ptr [[TMP4]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> poison, float [[TMP4]], i64 0 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] ; CHECK: pred.load.continue: -; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ] -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1 -; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] +; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1 +; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] ; CHECK: pred.load.if1: -; CHECK-NEXT: [[TMP9:%.*]] = or disjoint i64 [[INDEX]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]] -; CHECK-NEXT: [[TMP11:%.*]] = load float, ptr [[TMP10]], align 4 -; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x float> [[TMP7]], float [[TMP11]], i64 1 +; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x float> [[TMP6]], float [[TMP10]], i64 1 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]] ; CHECK: pred.load.continue2: -; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x float> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ] -; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2 -; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] +; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x float> [ [[TMP6]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], [[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2 +; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] ; CHECK: pred.load.if3: -; CHECK-NEXT: [[TMP15:%.*]] = or disjoint i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP15]] -; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[TMP16]], align 4 -; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x float> [[TMP13]], float [[TMP17]], i64 2 +; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP14]] +; CHECK-NEXT: [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4 +; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x float> [[TMP12]], float [[TMP16]], i64 2 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]] ; CHECK: pred.load.continue4: -; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x float> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ] -; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3 -; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]] +; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x float> [ [[TMP12]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP17]], [[PRED_LOAD_IF3]] ] +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3 +; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]] ; CHECK: pred.load.if5: -; CHECK-NEXT: [[TMP21:%.*]] = or disjoint i64 [[INDEX]], 3 -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = load float, ptr [[TMP22]], align 4 -; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x float> [[TMP19]], float [[TMP23]], i64 3 +; CHECK-NEXT: [[TMP20:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP20]] +; CHECK-NEXT: [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x float> [[TMP18]], float [[TMP22]], i64 3 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]] ; CHECK: pred.load.continue6: -; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x float> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ] -; CHECK-NEXT: [[TMP26:%.*]] = select fast <4 x i1> [[TMP2]], <4 x float> [[TMP25]], <4 x float> zeroinitializer -; CHECK-NEXT: [[TMP27]] = call fast float @llvm.vector.reduce.fadd.v4f32(float [[VEC_PHI]], <4 x float> [[TMP26]]) +; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x float> [ [[TMP18]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ] +; CHECK-NEXT: [[TMP25:%.*]] = select fast <4 x i1> [[TMP1]], <4 x float> [[TMP24]], <4 x float> zeroinitializer +; CHECK-NEXT: [[TMP26]] = call fast float @llvm.vector.reduce.fadd.v4f32(float [[VEC_PHI]], <4 x float> [[TMP25]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP27]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[ENTRY]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP26]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[ENTRY]] ] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ] ; CHECK-NEXT: [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RES:%.*]], [[FOR_INC]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[COND]], i64 [[IV]] -; CHECK-NEXT: [[TMP29:%.*]] = load float, ptr [[ARRAYIDX]], align 4 -; CHECK-NEXT: [[TOBOOL:%.*]] = fcmp une float [[TMP29]], 5.000000e+00 +; CHECK-NEXT: [[TMP28:%.*]] = load float, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[TOBOOL:%.*]] = fcmp une float [[TMP28]], 5.000000e+00 ; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]] ; CHECK: if.then: ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]] -; CHECK-NEXT: [[TMP30:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 -; CHECK-NEXT: [[FADD:%.*]] = fadd fast float [[RDX]], [[TMP30]] +; CHECK-NEXT: [[TMP29:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[FADD:%.*]] = fadd fast float [[RDX]], [[TMP29]] ; CHECK-NEXT: br label [[FOR_INC]] ; CHECK: for.inc: ; CHECK-NEXT: [[RES]] = phi float [ [[RDX]], [[FOR_BODY]] ], [ [[FADD]], [[IF_THEN]] ] ; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: for.end: -; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[TMP26]], [[MIDDLE_BLOCK]] ] ; CHECK-NEXT: ret float [[RES_LCSSA]] ; entry: @@ -114,90 +116,91 @@ for.end: } define float @cond_cmp_sel(ptr noalias %a, ptr noalias %cond, i64 %N) { -; CHECK-LABEL: @cond_cmp_sel( +; CHECK-LABEL: define float @cond_cmp_sel( +; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[COND:%.*]], i64 [[N:%.*]]) { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[PRED_LOAD_CONTINUE6]] ] -; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[COND:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[RDX_MINMAX_SELECT:%.*]], [[PRED_LOAD_CONTINUE6]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[COND]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = fcmp une <4 x float> [[WIDE_LOAD]], -; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0 -; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = fcmp une <4 x float> [[WIDE_LOAD]], +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0 +; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] ; CHECK: pred.load.if: -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[TMP5:%.*]] = load float, ptr [[TMP4]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> poison, float [[TMP4]], i64 0 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] ; CHECK: pred.load.continue: -; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ] -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1 -; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] +; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1 +; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] ; CHECK: pred.load.if1: -; CHECK-NEXT: [[TMP9:%.*]] = or disjoint i64 [[INDEX]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]] -; CHECK-NEXT: [[TMP11:%.*]] = load float, ptr [[TMP10]], align 4 -; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x float> [[TMP7]], float [[TMP11]], i64 1 +; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x float> [[TMP6]], float [[TMP10]], i64 1 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]] ; CHECK: pred.load.continue2: -; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x float> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ] -; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2 -; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] +; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x float> [ [[TMP6]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], [[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2 +; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] ; CHECK: pred.load.if3: -; CHECK-NEXT: [[TMP15:%.*]] = or disjoint i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP15]] -; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[TMP16]], align 4 -; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x float> [[TMP13]], float [[TMP17]], i64 2 +; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP14]] +; CHECK-NEXT: [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4 +; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x float> [[TMP12]], float [[TMP16]], i64 2 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]] ; CHECK: pred.load.continue4: -; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x float> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ] -; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3 -; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]] +; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x float> [ [[TMP12]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP17]], [[PRED_LOAD_IF3]] ] +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3 +; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]] ; CHECK: pred.load.if5: -; CHECK-NEXT: [[TMP21:%.*]] = or disjoint i64 [[INDEX]], 3 -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = load float, ptr [[TMP22]], align 4 -; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x float> [[TMP19]], float [[TMP23]], i64 3 +; CHECK-NEXT: [[TMP20:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP20]] +; CHECK-NEXT: [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x float> [[TMP18]], float [[TMP22]], i64 3 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]] ; CHECK: pred.load.continue6: -; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x float> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ] -; CHECK-NEXT: [[TMP26:%.*]] = select fast <4 x i1> [[TMP2]], <4 x float> [[TMP25]], <4 x float> -; CHECK-NEXT: [[TMP27:%.*]] = call fast float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP26]]) -; CHECK-NEXT: [[TMP28]] = call fast float @llvm.minnum.f32(float [[TMP27]], float [[VEC_PHI]]) +; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x float> [ [[TMP18]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ] +; CHECK-NEXT: [[TMP25:%.*]] = select fast <4 x i1> [[TMP1]], <4 x float> [[TMP24]], <4 x float> +; CHECK-NEXT: [[TMP26:%.*]] = call fast float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP25]]) +; CHECK-NEXT: [[RDX_MINMAX_SELECT]] = call fast float @llvm.minnum.f32(float [[TMP26]], float [[VEC_PHI]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP28]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[ENTRY]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[RDX_MINMAX_SELECT]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[ENTRY]] ] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ] ; CHECK-NEXT: [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RES:%.*]], [[FOR_INC]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[COND]], i64 [[IV]] -; CHECK-NEXT: [[TMP30:%.*]] = load float, ptr [[ARRAYIDX]], align 4 -; CHECK-NEXT: [[TOBOOL:%.*]] = fcmp une float [[TMP30]], 3.000000e+00 +; CHECK-NEXT: [[TMP28:%.*]] = load float, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[TOBOOL:%.*]] = fcmp une float [[TMP28]], 3.000000e+00 ; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]] ; CHECK: if.then: ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]] -; CHECK-NEXT: [[TMP31:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 -; CHECK-NEXT: [[TMP32:%.*]] = call fast float @llvm.minnum.f32(float [[RDX]], float [[TMP31]]) +; CHECK-NEXT: [[TMP29:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[FSEL:%.*]] = call fast float @llvm.minnum.f32(float [[RDX]], float [[TMP29]]) ; CHECK-NEXT: br label [[FOR_INC]] ; CHECK: for.inc: -; CHECK-NEXT: [[RES]] = phi float [ [[RDX]], [[FOR_BODY]] ], [ [[TMP32]], [[IF_THEN]] ] +; CHECK-NEXT: [[RES]] = phi float [ [[RDX]], [[FOR_BODY]] ], [ [[FSEL]], [[IF_THEN]] ] ; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] ; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: for.end: -; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[TMP28]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[RDX_MINMAX_SELECT]], [[MIDDLE_BLOCK]] ] ; CHECK-NEXT: ret float [[RES_LCSSA]] ; entry: @@ -229,84 +232,85 @@ for.end: } define i32 @conditional_and(ptr noalias %A, ptr noalias %B, i32 %cond, i64 noundef %N) #0 { -; CHECK-LABEL: @conditional_and( +; CHECK-LABEL: define i32 @conditional_and( +; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i32 [[COND:%.*]], i64 noundef [[N:%.*]]) { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[COND:%.*]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[COND]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 7, [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[PRED_LOAD_CONTINUE6]] ] -; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 7, [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[PRED_LOAD_CONTINUE6]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0 -; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0 +; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] ; CHECK: pred.load.if: -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] ; CHECK: pred.load.continue: -; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ] -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1 -; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] +; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1 +; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] ; CHECK: pred.load.if1: -; CHECK-NEXT: [[TMP9:%.*]] = or disjoint i64 [[INDEX]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP9]] -; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 -; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP11]], i64 1 +; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP10]], i64 1 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]] ; CHECK: pred.load.continue2: -; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ] -; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2 -; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] +; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ [[TMP6]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], [[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2 +; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] ; CHECK: pred.load.if3: -; CHECK-NEXT: [[TMP15:%.*]] = or disjoint i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP15]] -; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4 -; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP17]], i64 2 +; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP14]] +; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4 +; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP16]], i64 2 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]] ; CHECK: pred.load.continue4: -; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ] -; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3 -; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]] +; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP12]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP17]], [[PRED_LOAD_IF3]] ] +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3 +; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]] ; CHECK: pred.load.if5: -; CHECK-NEXT: [[TMP21:%.*]] = or disjoint i64 [[INDEX]], 3 -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4 -; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP23]], i64 3 +; CHECK-NEXT: [[TMP20:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP20]] +; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP22]], i64 3 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]] ; CHECK: pred.load.continue6: -; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x i32> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ] -; CHECK-NEXT: [[TMP26:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP25]], <4 x i32> -; CHECK-NEXT: [[TMP27:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP26]]) -; CHECK-NEXT: [[TMP28]] = and i32 [[TMP27]], [[VEC_PHI]] +; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i32> [ [[TMP18]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ] +; CHECK-NEXT: [[TMP25:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP24]], <4 x i32> +; CHECK-NEXT: [[TMP26:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP25]]) +; CHECK-NEXT: [[TMP27]] = and i32 [[TMP26]], [[VEC_PHI]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP28]], [[MIDDLE_BLOCK]] ], [ 7, [[ENTRY]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP27]], [[MIDDLE_BLOCK]] ], [ 7, [[ENTRY]] ] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ] ; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RES:%.*]], [[FOR_INC]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]] -; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 -; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[TMP30]], [[COND]] +; CHECK-NEXT: [[TMP29:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[TMP29]], [[COND]] ; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]] ; CHECK: if.then: ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] -; CHECK-NEXT: [[TMP31:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 -; CHECK-NEXT: [[AND:%.*]] = and i32 [[TMP31]], [[RDX]] +; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[AND:%.*]] = and i32 [[TMP30]], [[RDX]] ; CHECK-NEXT: br label [[FOR_INC]] ; CHECK: for.inc: ; CHECK-NEXT: [[RES]] = phi i32 [ [[AND]], [[IF_THEN]] ], [ [[RDX]], [[FOR_BODY]] ] @@ -314,7 +318,7 @@ define i32 @conditional_and(ptr noalias %A, ptr noalias %B, i32 %cond, i64 nound ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] ; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK: for.end: -; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i32 [ [[RES]], [[FOR_INC]] ], [ [[TMP28]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i32 [ [[RES]], [[FOR_INC]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ] ; CHECK-NEXT: ret i32 [[RES_LCSSA]] ; entry: @@ -345,9 +349,10 @@ for.end: } define i32 @simple_chained_rdx(ptr noalias %a, ptr noalias %b, ptr noalias %cond, i64 noundef %N) { -; CHECK-LABEL: @simple_chained_rdx( +; CHECK-LABEL: define i32 @simple_chained_rdx( +; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[COND:%.*]], i64 noundef [[N:%.*]]) { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 @@ -355,16 +360,16 @@ define i32 @simple_chained_rdx(ptr noalias %a, ptr noalias %b, ptr noalias %cond ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ] ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 5, [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[PRED_LOAD_CONTINUE6]] ] -; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer ; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0 ; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] ; CHECK: pred.load.if: -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]] ; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 ; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] ; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 ; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i64 0 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] @@ -491,12 +496,140 @@ for.end: ; Reduction not performed in loop as the phi has more than two incoming values ; define i64 @nested_cond_and(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture readonly %cond, i64 %N){ -; CHECK-LABEL: @nested_cond_and( +; CHECK-LABEL: define i64 @nested_cond_and( +; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], ptr noalias nocapture readonly [[COND:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: -; CHECK-NOT: @llvm.vector.reduce.and +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE14:%.*]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ , [[VECTOR_PH]] ], [ [[PREDPHI15:%.*]], [[PRED_LOAD_CONTINUE14]] ] +; CHECK-NEXT: [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0 +; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK: pred.load.if: +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 4 +; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i64> poison, i64 [[TMP7]], i64 0 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] +; CHECK: pred.load.continue: +; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i64> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1 +; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] +; CHECK: pred.load.if1: +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]] +; CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP11]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> [[TMP9]], i64 [[TMP12]], i64 1 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]] +; CHECK: pred.load.continue2: +; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i64> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2 +; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] +; CHECK: pred.load.if3: +; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]] +; CHECK-NEXT: [[TMP17:%.*]] = load i64, ptr [[TMP16]], align 4 +; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP17]], i64 2 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]] +; CHECK: pred.load.continue4: +; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x i64> [ [[TMP14]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ] +; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3 +; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]] +; CHECK: pred.load.if5: +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]] +; CHECK-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> [[TMP19]], i64 [[TMP22]], i64 3 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]] +; CHECK: pred.load.continue6: +; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i64> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ] +; CHECK-NEXT: [[TMP25:%.*]] = icmp eq <4 x i64> [[TMP24]], +; CHECK-NEXT: [[TMP26:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP25]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP26]], i64 0 +; CHECK-NEXT: br i1 [[TMP27]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]] +; CHECK: pred.load.if7: +; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP28]], align 4 +; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i64> poison, i64 [[TMP29]], i64 0 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE8]] +; CHECK: pred.load.continue8: +; CHECK-NEXT: [[TMP31:%.*]] = phi <4 x i64> [ poison, [[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], [[PRED_LOAD_IF7]] ] +; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP26]], i64 1 +; CHECK-NEXT: br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]] +; CHECK: pred.load.if9: +; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]] +; CHECK-NEXT: [[TMP34:%.*]] = load i64, ptr [[TMP33]], align 4 +; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x i64> [[TMP31]], i64 [[TMP34]], i64 1 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE10]] +; CHECK: pred.load.continue10: +; CHECK-NEXT: [[TMP36:%.*]] = phi <4 x i64> [ [[TMP31]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP35]], [[PRED_LOAD_IF9]] ] +; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[TMP26]], i64 2 +; CHECK-NEXT: br i1 [[TMP37]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]] +; CHECK: pred.load.if11: +; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]] +; CHECK-NEXT: [[TMP39:%.*]] = load i64, ptr [[TMP38]], align 4 +; CHECK-NEXT: [[TMP40:%.*]] = insertelement <4 x i64> [[TMP36]], i64 [[TMP39]], i64 2 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE12]] +; CHECK: pred.load.continue12: +; CHECK-NEXT: [[TMP41:%.*]] = phi <4 x i64> [ [[TMP36]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP40]], [[PRED_LOAD_IF11]] ] +; CHECK-NEXT: [[TMP42:%.*]] = extractelement <4 x i1> [[TMP26]], i64 3 +; CHECK-NEXT: br i1 [[TMP42]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14]] +; CHECK: pred.load.if13: +; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]] +; CHECK-NEXT: [[TMP44:%.*]] = load i64, ptr [[TMP43]], align 4 +; CHECK-NEXT: [[TMP45:%.*]] = insertelement <4 x i64> [[TMP41]], i64 [[TMP44]], i64 3 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE14]] +; CHECK: pred.load.continue14: +; CHECK-NEXT: [[TMP46:%.*]] = phi <4 x i64> [ [[TMP41]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP45]], [[PRED_LOAD_IF13]] ] +; CHECK-NEXT: [[TMP47:%.*]] = xor <4 x i1> [[TMP25]], +; CHECK-NEXT: [[TMP48:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP47]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[PREDPHI_V:%.*]] = select <4 x i1> [[TMP48]], <4 x i64> [[TMP24]], <4 x i64> [[TMP46]] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> [[PREDPHI_V]], <4 x i64> +; CHECK-NEXT: [[PREDPHI15]] = and <4 x i64> [[VEC_PHI]], [[PREDPHI]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP49:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP49]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: middle.block: -; CHECK: @llvm.vector.reduce.and -; CHECK: scalar.ph +; CHECK-NEXT: [[TMP50:%.*]] = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> [[PREDPHI15]]) +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP50]], [[MIDDLE_BLOCK]] ], [ 5, [[ENTRY]] ] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ] +; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RES:%.*]], [[FOR_INC]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[COND]], i64 [[IV]] +; CHECK-NEXT: [[TMP51:%.*]] = load i64, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i64 [[TMP51]], 0 +; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]] +; CHECK: if.then: +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] +; CHECK-NEXT: [[TMP52:%.*]] = load i64, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[AND1:%.*]] = and i64 [[RDX]], [[TMP52]] +; CHECK-NEXT: [[TOBOOL2:%.*]] = icmp eq i64 [[TMP52]], 3 +; CHECK-NEXT: br i1 [[TOBOOL2]], label [[IF_THEN_2:%.*]], label [[FOR_INC]] +; CHECK: if.then.2: +; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]] +; CHECK-NEXT: [[TMP53:%.*]] = load i64, ptr [[ARRAYIDX3]], align 4 +; CHECK-NEXT: [[AND2:%.*]] = and i64 [[RDX]], [[TMP53]] +; CHECK-NEXT: br label [[FOR_INC]] +; CHECK: for.inc: +; CHECK-NEXT: [[RES]] = phi i64 [ [[AND2]], [[IF_THEN_2]] ], [ [[AND1]], [[IF_THEN]] ], [ [[RDX]], [[FOR_BODY]] ] +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i64 [ [[RES]], [[FOR_INC]] ], [ [[TMP50]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret i64 [[RES_LCSSA]] +; entry: br label %for.body @@ -536,12 +669,99 @@ for.end: ; the Phi (%rdx1) as it has more than one use. ; define i32 @cond-uncond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond, i64 noundef %n) #0 { -; CHECK-LABEL: @cond-uncond( +; CHECK-LABEL: define i32 @cond-uncond( +; CHECK-SAME: ptr noalias [[SRC1:%.*]], ptr noalias [[SRC2:%.*]], ptr noalias [[COND:%.*]], i64 noundef [[N:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[PRED_LOAD_CONTINUE6]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0 +; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK: pred.load.if: +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] +; CHECK: pred.load.continue: +; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1 +; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] +; CHECK: pred.load.if1: +; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP10]], i64 1 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]] +; CHECK: pred.load.continue2: +; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ [[TMP6]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], [[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2 +; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] +; CHECK: pred.load.if3: +; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP14]] +; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4 +; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP16]], i64 2 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]] +; CHECK: pred.load.continue4: +; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP12]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP17]], [[PRED_LOAD_IF3]] ] +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3 +; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]] +; CHECK: pred.load.if5: +; CHECK-NEXT: [[TMP20:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP20]] +; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP22]], i64 3 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]] ; CHECK: pred.load.continue6: -; CHECK-NOT: @llvm.vector.reduce.add +; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i32> [ [[TMP18]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ] +; CHECK-NEXT: [[TMP25:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP24]], <4 x i32> zeroinitializer +; CHECK-NEXT: [[PREDPHI:%.*]] = add <4 x i32> [[VEC_PHI]], [[TMP25]] +; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP26]], align 4 +; CHECK-NEXT: [[TMP27]] = add <4 x i32> [[WIDE_LOAD7]], [[PREDPHI]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] ; CHECK: middle.block: -; CHECK: @llvm.vector.reduce.add -; CHECK: scalar.ph +; CHECK-NEXT: [[TMP29:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP27]]) +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP29]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[RDX1:%.*]] = phi i32 [ [[ADD2:%.*]], [[IF_END:%.*]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[IF_END]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[IV]] +; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP30]], 0 +; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[IV]] +; CHECK-NEXT: [[TMP31:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4 +; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP31]], [[RDX1]] +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[RDX1]], [[FOR_BODY]] ] +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[IV]] +; CHECK-NEXT: [[TMP32:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[ADD2]] = add nsw i32 [[TMP32]], [[RES]] +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: [[ADD2_LCSSA:%.*]] = phi i32 [ [[ADD2]], [[IF_END]] ], [ [[TMP29]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret i32 [[ADD2_LCSSA]] +; entry: br label %for.body @@ -577,12 +797,141 @@ for.end: ; of the incoming values of the LoopExitInstruction (%res) is the reduction Phi (%rdx1). ; define float @cond_cond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond, i64 %n) #0 { -; CHECK-LABEL: @cond_cond( +; CHECK-LABEL: define float @cond_cond( +; CHECK-SAME: ptr noalias [[SRC1:%.*]], ptr noalias [[SRC2:%.*]], ptr noalias [[COND:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE14:%.*]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ , [[VECTOR_PH]] ], [ [[PREDPHI15:%.*]], [[PRED_LOAD_CONTINUE14]] ] +; CHECK-NEXT: [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = fcmp fast oeq <4 x float> [[WIDE_LOAD]], +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0 +; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK: pred.load.if: +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP7:%.*]] = load float, ptr [[TMP6]], align 4 +; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> poison, float [[TMP7]], i64 0 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] +; CHECK: pred.load.continue: +; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1 +; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] +; CHECK: pred.load.if1: +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[TMP0]] +; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x float> [[TMP9]], float [[TMP12]], i64 1 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]] +; CHECK: pred.load.continue2: +; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x float> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2 +; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] +; CHECK: pred.load.if3: +; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[TMP1]] +; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[TMP16]], align 4 +; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x float> [[TMP14]], float [[TMP17]], i64 2 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]] +; CHECK: pred.load.continue4: +; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x float> [ [[TMP14]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ] +; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3 +; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]] +; CHECK: pred.load.if5: +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[TMP2]] +; CHECK-NEXT: [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x float> [[TMP19]], float [[TMP22]], i64 3 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]] +; CHECK: pred.load.continue6: +; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x float> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ] +; CHECK-NEXT: [[TMP25:%.*]] = select <4 x i1> [[TMP4]], <4 x float> [[TMP24]], <4 x float> +; CHECK-NEXT: [[PREDPHI:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[TMP25]] +; CHECK-NEXT: [[TMP26:%.*]] = fcmp fast oeq <4 x float> [[WIDE_LOAD]], +; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP26]], i64 0 +; CHECK-NEXT: br i1 [[TMP27]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]] +; CHECK: pred.load.if7: +; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP29:%.*]] = load float, ptr [[TMP28]], align 4 +; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x float> poison, float [[TMP29]], i64 0 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE8]] +; CHECK: pred.load.continue8: +; CHECK-NEXT: [[TMP31:%.*]] = phi <4 x float> [ poison, [[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], [[PRED_LOAD_IF7]] ] +; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP26]], i64 1 +; CHECK-NEXT: br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]] +; CHECK: pred.load.if9: +; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[TMP0]] +; CHECK-NEXT: [[TMP34:%.*]] = load float, ptr [[TMP33]], align 4 +; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x float> [[TMP31]], float [[TMP34]], i64 1 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE10]] +; CHECK: pred.load.continue10: +; CHECK-NEXT: [[TMP36:%.*]] = phi <4 x float> [ [[TMP31]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP35]], [[PRED_LOAD_IF9]] ] +; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[TMP26]], i64 2 +; CHECK-NEXT: br i1 [[TMP37]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]] +; CHECK: pred.load.if11: +; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[TMP1]] +; CHECK-NEXT: [[TMP39:%.*]] = load float, ptr [[TMP38]], align 4 +; CHECK-NEXT: [[TMP40:%.*]] = insertelement <4 x float> [[TMP36]], float [[TMP39]], i64 2 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE12]] +; CHECK: pred.load.continue12: +; CHECK-NEXT: [[TMP41:%.*]] = phi <4 x float> [ [[TMP36]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP40]], [[PRED_LOAD_IF11]] ] +; CHECK-NEXT: [[TMP42:%.*]] = extractelement <4 x i1> [[TMP26]], i64 3 +; CHECK-NEXT: br i1 [[TMP42]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14]] +; CHECK: pred.load.if13: +; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[TMP2]] +; CHECK-NEXT: [[TMP44:%.*]] = load float, ptr [[TMP43]], align 4 +; CHECK-NEXT: [[TMP45:%.*]] = insertelement <4 x float> [[TMP41]], float [[TMP44]], i64 3 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE14]] ; CHECK: pred.load.continue14: -; CHECK-NOT: @llvm.vector.reduce.fadd +; CHECK-NEXT: [[TMP46:%.*]] = phi <4 x float> [ [[TMP41]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP45]], [[PRED_LOAD_IF13]] ] +; CHECK-NEXT: [[TMP47:%.*]] = select <4 x i1> [[TMP26]], <4 x float> [[TMP46]], <4 x float> +; CHECK-NEXT: [[PREDPHI15]] = fadd fast <4 x float> [[PREDPHI]], [[TMP47]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP48]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] ; CHECK: middle.block: -; CHECK: @llvm.vector.reduce.fadd -; CHECK: scalar.ph +; CHECK-NEXT: [[TMP49:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PREDPHI15]]) +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP49]], [[MIDDLE_BLOCK]] ], [ 2.000000e+00, [[ENTRY]] ] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[RDX1:%.*]] = phi float [ [[RES:%.*]], [[FOR_INC:%.*]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[COND]], i64 [[IV]] +; CHECK-NEXT: [[TMP50:%.*]] = load float, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[CMP1:%.*]] = fcmp fast oeq float [[TMP50]], 3.000000e+00 +; CHECK-NEXT: br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[IV]] +; CHECK-NEXT: [[TMP51:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP51]], [[RDX1]] +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: [[RDX2:%.*]] = phi float [ [[ADD]], [[IF_THEN]] ], [ [[RDX1]], [[FOR_BODY]] ] +; CHECK-NEXT: [[CMP5:%.*]] = fcmp fast oeq float [[TMP50]], 7.000000e+00 +; CHECK-NEXT: br i1 [[CMP5]], label [[IF_THEN6:%.*]], label [[FOR_INC]] +; CHECK: if.then6: +; CHECK-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[IV]] +; CHECK-NEXT: [[TMP52:%.*]] = load float, ptr [[ARRAYIDX7]], align 4 +; CHECK-NEXT: [[ADD2:%.*]] = fadd fast float [[TMP52]], [[RDX2]] +; CHECK-NEXT: br label [[FOR_INC]] +; CHECK: for.inc: +; CHECK-NEXT: [[RES]] = phi float [ [[ADD2]], [[IF_THEN6]] ], [ [[RDX2]], [[IF_END]] ] +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[TMP49]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret float [[RES_LCSSA]] +; entry: br label %for.body @@ -626,12 +975,99 @@ for.end: ; incoming values of the LoopExitInstruction (%res) is the reduction Phi (%rdx). ; define i32 @uncond_cond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond, i64 %N) #0 { -; CHECK-LABEL: @uncond_cond( +; CHECK-LABEL: define i32 @uncond_cond( +; CHECK-SAME: ptr noalias [[SRC1:%.*]], ptr noalias [[SRC2:%.*]], ptr noalias [[COND:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE7:%.*]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PREDPHI:%.*]], [[PRED_LOAD_CONTINUE7]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]] +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD1]], zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0 +; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK: pred.load.if: +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i64 0 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] +; CHECK: pred.load.continue: +; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1 +; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3:%.*]] +; CHECK: pred.load.if2: +; CHECK-NEXT: [[TMP10:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP12]], i64 1 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE3]] +; CHECK: pred.load.continue3: +; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP8]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF2]] ] +; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2 +; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5:%.*]] +; CHECK: pred.load.if4: +; CHECK-NEXT: [[TMP16:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP16]] +; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4 +; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP18]], i64 2 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE5]] +; CHECK: pred.load.continue5: +; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ [[TMP14]], [[PRED_LOAD_CONTINUE3]] ], [ [[TMP19]], [[PRED_LOAD_IF4]] ] +; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3 +; CHECK-NEXT: br i1 [[TMP21]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7]] +; CHECK: pred.load.if6: +; CHECK-NEXT: [[TMP22:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP22]] +; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4 +; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP24]], i64 3 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE7]] ; CHECK: pred.load.continue7: -; CHECK-NOT: @llvm.vector.reduce.add +; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ [[TMP20]], [[PRED_LOAD_CONTINUE5]] ], [ [[TMP25]], [[PRED_LOAD_IF6]] ] +; CHECK-NEXT: [[TMP27:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP26]], <4 x i32> zeroinitializer +; CHECK-NEXT: [[PREDPHI]] = add <4 x i32> [[TMP1]], [[TMP27]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] ; CHECK: middle.block: -; CHECK: @llvm.vector.reduce.add -; CHECK: scalar.ph +; CHECK-NEXT: [[TMP29:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PREDPHI]]) +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP29]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ [[RES:%.*]], [[FOR_INC:%.*]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[IV]] +; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[ADD1:%.*]] = add nsw i32 [[TMP30]], [[RDX]] +; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[IV]] +; CHECK-NEXT: [[TMP31:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4 +; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP31]], 0 +; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label [[FOR_INC]], label [[IF_THEN:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[IV]] +; CHECK-NEXT: [[TMP32:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[ADD2:%.*]] = add nsw i32 [[TMP32]], [[ADD1]] +; CHECK-NEXT: br label [[FOR_INC]] +; CHECK: for.inc: +; CHECK-NEXT: [[RES]] = phi i32 [ [[ADD2]], [[IF_THEN]] ], [ [[ADD1]], [[FOR_BODY]] ] +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i32 [ [[RES]], [[FOR_INC]] ], [ [[TMP29]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret i32 [[RES_LCSSA]] +; entry: br label %for.body @@ -667,12 +1103,101 @@ for.end: ; through the chain and check the number of uses of %add1, we find more than the expected one use. ; define i32 @uncond_cond_uncond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond, i64 noundef %N) { -; CHECK-LABEL: @uncond_cond_uncond( +; CHECK-LABEL: define i32 @uncond_cond_uncond( +; CHECK-SAME: ptr noalias [[SRC1:%.*]], ptr noalias [[SRC2:%.*]], ptr noalias [[COND:%.*]], i64 noundef [[N:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE7:%.*]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[PRED_LOAD_CONTINUE7]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]] +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD1]], zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0 +; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK: pred.load.if: +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i64 0 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] +; CHECK: pred.load.continue: +; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1 +; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3:%.*]] +; CHECK: pred.load.if2: +; CHECK-NEXT: [[TMP10:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP12]], i64 1 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE3]] +; CHECK: pred.load.continue3: +; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP8]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF2]] ] +; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2 +; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5:%.*]] +; CHECK: pred.load.if4: +; CHECK-NEXT: [[TMP16:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP16]] +; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4 +; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP18]], i64 2 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE5]] +; CHECK: pred.load.continue5: +; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ [[TMP14]], [[PRED_LOAD_CONTINUE3]] ], [ [[TMP19]], [[PRED_LOAD_IF4]] ] +; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3 +; CHECK-NEXT: br i1 [[TMP21]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7]] +; CHECK: pred.load.if6: +; CHECK-NEXT: [[TMP22:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP22]] +; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4 +; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP24]], i64 3 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE7]] ; CHECK: pred.load.continue7: -; CHECK-NOT: @llvm.vector.reduce.add +; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ [[TMP20]], [[PRED_LOAD_CONTINUE5]] ], [ [[TMP25]], [[PRED_LOAD_IF6]] ] +; CHECK-NEXT: [[TMP27:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP26]], <4 x i32> zeroinitializer +; CHECK-NEXT: [[PREDPHI:%.*]] = add <4 x i32> [[TMP1]], [[TMP27]] +; CHECK-NEXT: [[TMP28]] = add <4 x i32> [[PREDPHI]], [[WIDE_LOAD]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] ; CHECK: middle.block: -; CHECK: @llvm.vector.reduce.add -; CHECK: scalar.ph +; CHECK-NEXT: [[TMP30:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP28]]) +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP30]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ [[ADD3:%.*]], [[IF_END:%.*]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[IF_END]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[IV]] +; CHECK-NEXT: [[TMP31:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[ADD1:%.*]] = add nsw i32 [[TMP31]], [[RDX]] +; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[IV]] +; CHECK-NEXT: [[TMP32:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4 +; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP32]], 0 +; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[IV]] +; CHECK-NEXT: [[TMP33:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[ADD2:%.*]] = add nsw i32 [[TMP33]], [[ADD1]] +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[ADD2]], [[IF_THEN]] ], [ [[ADD1]], [[FOR_BODY]] ] +; CHECK-NEXT: [[ADD3]] = add nsw i32 [[RES]], [[TMP31]] +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: [[ADD3_LCSSA:%.*]] = phi i32 [ [[ADD3]], [[IF_END]] ], [ [[TMP30]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret i32 [[ADD3_LCSSA]] +; entry: br label %for.body diff --git a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll index 3973a288fe71fb03e80ac4fef13595ac9fbd4532..2a58748d8fb67dc927c96afa6d58294db420b9e5 100644 --- a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll +++ b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll @@ -11,15 +11,12 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) { ; CHECK: vector.ph: ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 4 ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]] -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP0:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i32> undef, <4 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[VEC_PHI]], ; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[BROADCAST_SPLAT2]] ; CHECK-NEXT: [[TMP3:%.*]] = trunc <4 x i32> [[TMP2]] to <4 x i8> @@ -40,7 +37,7 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) { ; CHECK: for.body: ; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ] ; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[R_NEXT:%.*]], [[IF_END]] ] -; CHECK-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]] +; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_END]] ; CHECK: if.then: ; CHECK-NEXT: [[T0:%.*]] = sdiv i32 undef, undef ; CHECK-NEXT: br label [[IF_END]] diff --git a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll index 6d059c13597be091736140bafdc08242cfb257fe..3be31c011eaacf394e5a2343681757457aedb18f 100644 --- a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll +++ b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll @@ -197,6 +197,7 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x) { ; CHECK-VF4UF2: vector.ph ; CHECK-VF4UF2: call i32 @llvm.vscale.i32() ; CHECK-VF4UF2: call i32 @llvm.vscale.i32() +; CHECK-VF4UF2: call i32 @llvm.vscale.i32() ; CHECK-VF4UF2: %[[VSCALE1:.*]] = call i32 @llvm.vscale.i32() ; CHECK-VF4UF2: %[[MUL1:.*]] = mul i32 %[[VSCALE1]], 4 ; CHECK-VF4UF2: %[[SUB1:.*]] = sub i32 %[[MUL1]], 1 diff --git a/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll b/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll index 034400ca2b0fb9a94f882cf29405459d0ad5b373..2d5db6fc28332366ac44cadcc8472dfbb700f4b4 100644 --- a/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll +++ b/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll @@ -19,33 +19,33 @@ define void @add_ind64_unrolled(ptr noalias nocapture %a, ptr noalias nocapture ; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] -; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i64() -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 1 -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP6]], i64 0 +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[TMP4]], 2 +; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv2i64() +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[TMP7]], 1 +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP8]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[STEP_ADD:%.*]] = add [[VEC_IND]], [[DOTSPLAT]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP7]], align 8 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[TMP8]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i64 [[TMP9]] -; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load , ptr [[TMP10]], align 8 -; CHECK-NEXT: [[TMP11:%.*]] = add nsw [[WIDE_LOAD]], [[VEC_IND]] -; CHECK-NEXT: [[TMP12:%.*]] = add nsw [[WIDE_LOAD2]], [[STEP_ADD]] -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: store [[TMP11]], ptr [[TMP13]], align 8 -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl i64 [[TMP14]], 1 -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 [[TMP15]] -; CHECK-NEXT: store [[TMP12]], ptr [[TMP16]], align 8 -; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP18:%.*]] = shl i64 [[TMP17]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP9]], align 8 +; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP11:%.*]] = shl i64 [[TMP10]], 1 +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP9]], i64 [[TMP11]] +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load , ptr [[TMP12]], align 8 +; CHECK-NEXT: [[TMP13:%.*]] = add nsw [[WIDE_LOAD]], [[VEC_IND]] +; CHECK-NEXT: [[TMP14:%.*]] = add nsw [[WIDE_LOAD2]], [[STEP_ADD]] +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: store [[TMP13]], ptr [[TMP15]], align 8 +; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP17:%.*]] = shl i64 [[TMP16]], 1 +; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP15]], i64 [[TMP17]] +; CHECK-NEXT: store [[TMP14]], ptr [[TMP18]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[STEP_ADD]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -101,30 +101,30 @@ define void @add_ind64_unrolled_nxv1i64(ptr noalias nocapture %a, ptr noalias no ; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 1 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] -; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv1i64() -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[TMP4]], 1 +; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv1i64() +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP7]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[STEP_ADD:%.*]] = add [[VEC_IND]], [[DOTSPLAT]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP7]] -; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load , ptr [[TMP8]], align 8 -; CHECK-NEXT: [[TMP9:%.*]] = add nsw [[WIDE_LOAD]], [[VEC_IND]] -; CHECK-NEXT: [[TMP10:%.*]] = add nsw [[WIDE_LOAD2]], [[STEP_ADD]] -; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: store [[TMP9]], ptr [[TMP11]], align 8 -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i64 [[TMP12]] -; CHECK-NEXT: store [[TMP10]], ptr [[TMP13]], align 8 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i64 [[TMP9]] +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load , ptr [[TMP10]], align 8 +; CHECK-NEXT: [[TMP11:%.*]] = add nsw [[WIDE_LOAD]], [[VEC_IND]] +; CHECK-NEXT: [[TMP12:%.*]] = add nsw [[WIDE_LOAD2]], [[STEP_ADD]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: store [[TMP11]], ptr [[TMP13]], align 8 ; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl i64 [[TMP14]], 1 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 [[TMP14]] +; CHECK-NEXT: store [[TMP12]], ptr [[TMP15]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[STEP_ADD]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] @@ -189,21 +189,21 @@ define void @add_unique_ind32(ptr noalias nocapture %a, i64 %n) { ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] ; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32 ; CHECK-NEXT: [[IND_END:%.*]] = shl i32 [[DOTCAST]], 1 -; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv4i32() -; CHECK-NEXT: [[TMP5:%.*]] = shl [[TMP4]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP7:%.*]] = shl i32 [[TMP6]], 3 -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i32 [[TMP7]], i64 0 +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[TMP4]], 2 +; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv4i32() +; CHECK-NEXT: [[TMP7:%.*]] = shl [[TMP6]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP9:%.*]] = shl i32 [[TMP8]], 3 +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i32 [[TMP9]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: store [[VEC_IND]], ptr [[TMP8]], align 4 -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP10:%.*]] = shl i64 [[TMP9]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP7]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: store [[VEC_IND]], ptr [[TMP10]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] @@ -267,25 +267,25 @@ define void @add_unique_indf32(ptr noalias nocapture %a, i64 %n) { ; CHECK-NEXT: [[DOTCAST:%.*]] = sitofp i64 [[N_VEC]] to float ; CHECK-NEXT: [[TMP4:%.*]] = fmul float [[DOTCAST]], 2.000000e+00 ; CHECK-NEXT: [[IND_END:%.*]] = fadd float [[TMP4]], 0.000000e+00 -; CHECK-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv4i32() -; CHECK-NEXT: [[TMP6:%.*]] = uitofp [[TMP5]] to -; CHECK-NEXT: [[TMP7:%.*]] = fmul [[TMP6]], shufflevector ( insertelement ( poison, float 2.000000e+00, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[INDUCTION:%.*]] = fadd [[TMP7]], zeroinitializer -; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP9:%.*]] = shl i32 [[TMP8]], 2 -; CHECK-NEXT: [[TMP10:%.*]] = uitofp i32 [[TMP9]] to float -; CHECK-NEXT: [[TMP11:%.*]] = fmul float [[TMP10]], 2.000000e+00 -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, float [[TMP11]], i64 0 +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 +; CHECK-NEXT: [[TMP7:%.*]] = call @llvm.experimental.stepvector.nxv4i32() +; CHECK-NEXT: [[TMP8:%.*]] = uitofp [[TMP7]] to +; CHECK-NEXT: [[TMP9:%.*]] = fmul [[TMP8]], shufflevector ( insertelement ( poison, float 2.000000e+00, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[INDUCTION:%.*]] = fadd [[TMP9]], zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP11:%.*]] = shl i32 [[TMP10]], 2 +; CHECK-NEXT: [[TMP12:%.*]] = uitofp i32 [[TMP11]] to float +; CHECK-NEXT: [[TMP13:%.*]] = fmul float [[TMP12]], 2.000000e+00 +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, float [[TMP13]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: store [[VEC_IND]], ptr [[TMP12]], align 4 -; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP14:%.*]] = shl i64 [[TMP13]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP14]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: store [[VEC_IND]], ptr [[TMP14]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = fadd [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll b/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll index 375d2e0ac8af59b5d339ff5538fb59f586c96bd0..b14d5a9f4bb5766ff6c2fbd3e63e37cb5cfce656 100644 --- a/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll +++ b/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll @@ -20,18 +20,18 @@ define void @test(ptr %d) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 128, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 128, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 ; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4096, ptr [[ARR]]) -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0 -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i32 100, i64 0), poison, zeroinitializer), ptr [[TMP6]], align 8 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0 +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i32 100, i64 0), poison, zeroinitializer), ptr [[TMP8]], align 8 ; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4096, ptr [[ARR]]) -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -92,18 +92,18 @@ define void @testloopvariant(ptr %d) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 128, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 128, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 ; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4096, ptr [[ARR]]) -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0 -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i32 100, i64 0), poison, zeroinitializer), ptr [[TMP6]], align 8 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0 +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i32 100, i64 0), poison, zeroinitializer), ptr [[TMP8]], align 8 ; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4096, ptr [[ARR]]) -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll b/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll index 6de52c09665c16cdc4072afa2d7a14a31a307354..33248e13320bfc80ac292299bd5f45a6d74f6186 100644 --- a/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll +++ b/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll @@ -12,6 +12,8 @@ ; CHECKUF1-DAG: %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2 ; CHECKUF1-DAG: %n.mod.vf = urem i64 %wide.trip.count, %[[VSCALEX4]] ; CHECKUF1: %n.vec = sub nsw i64 %wide.trip.count, %n.mod.vf +; CHECKUF1: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64() +; CHECKUF1: %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2 ; CHECKUF1: vector.body: ; CHECKUF1: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] @@ -20,8 +22,6 @@ ; CHECKUF1: %[[FADD:.*]] = fadd %wide.load, shufflevector ( insertelement ( poison, double 1.000000e+00, i64 0), poison, zeroinitializer) ; CHECKUF1: %[[IDXA:.*]] = getelementptr inbounds double, ptr %a, i64 %index ; CHECKUF1: store %[[FADD]], ptr %[[IDXA]], align 8 -; CHECKUF1: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64() -; CHECKUF1: %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2 ; CHECKUF1: %index.next = add nuw i64 %index, %[[VSCALEX4]] ; CHECKUF1: %[[CMP:.*]] = icmp eq i64 %index.next, %n.vec ; CHECKUF1: br i1 %[[CMP]], label %middle.block, label %vector.body, !llvm.loop !0 @@ -41,6 +41,8 @@ ; CHECKUF2-DAG: %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3 ; CHECKUF2-DAG: %n.mod.vf = urem i64 %wide.trip.count, %[[VSCALEX8]] ; CHECKUF2: %n.vec = sub nsw i64 %wide.trip.count, %n.mod.vf +; CHECKUF2: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64() +; CHECKUF2: %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3 ; CHECKUF2: vector.body: ; CHECKUF2: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] @@ -58,8 +60,6 @@ ; CHECKUF2: %[[VSCALE2:.*]] = shl i64 %[[VSCALE]], 2 ; CHECKUF2: %[[IDXA_NEXT:.*]] = getelementptr inbounds double, ptr %[[IDXA]], i64 %[[VSCALE2]] ; CHECKUF2: store %[[FADD_NEXT]], ptr %[[IDXA_NEXT]], align 8 -; CHECKUF2: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64() -; CHECKUF2: %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3 ; CHECKUF2: %index.next = add nuw i64 %index, %[[VSCALEX8]] ; CHECKUF2: %[[CMP:.*]] = icmp eq i64 %index.next, %n.vec ; CHECKUF2: br i1 %[[CMP]], label %middle.block, label %vector.body, !llvm.loop !0 diff --git a/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll index afe16c71f7f9ca492074a3eff50224e3d93707d8..e68e658f0e879b464b77e15869e33f988ac49191 100644 --- a/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll +++ b/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll @@ -5,6 +5,10 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 define i8 @reduction_add_trunc(ptr noalias nocapture %A) { ; CHECK-LABEL: @reduction_add_trunc( +; CHECK: call i32 @llvm.vscale.i32() +; CHECK: call i32 @llvm.vscale.i32() +; CHECK: [[TMP30:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP31:%.*]] = mul i32 [[TMP30]], 16 ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi [ insertelement ( zeroinitializer, i32 255, i32 0), %vector.ph ], [ [[TMP34:%.*]], %vector.body ] @@ -21,8 +25,6 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) { ; CHECK-NEXT: [[TMP35:%.*]] = trunc [[TMP29]] to ; CHECK-NEXT: [[TMP34]] = zext [[TMP33]] to ; CHECK-NEXT: [[TMP36]] = zext [[TMP35]] to -; CHECK-NEXT: [[TMP30:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP31:%.*]] = mul i32 [[TMP30]], 16 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP31]] ; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], {{%.*}} ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll b/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll index 8c92074abfd82ac172e2fe445258d0717ca32d03..18d2323ed6f5bc7e2d81ab05ec356060fb657b94 100644 --- a/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll +++ b/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll @@ -13,6 +13,8 @@ define void @trunc_minimal_bitwidth(ptr %bptr, ptr noalias %hptr, i32 %val, i64 ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[VAL:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -21,8 +23,6 @@ define void @trunc_minimal_bitwidth(ptr %bptr, ptr noalias %hptr, i32 %val, i64 ; CHECK-NEXT: [[TMP4:%.*]] = trunc [[BROADCAST_SPLAT]] to ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[HPTR:%.*]], i64 [[INDEX]] ; CHECK-NEXT: store [[TMP4]], ptr [[TMP5]], align 2 -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -73,6 +73,8 @@ define void @trunc_minimal_bitwidths_shufflevector (ptr %p, i32 %arg1, i64 %len) ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[LEN]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[LEN]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[ARG1:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: [[TMP4:%.*]] = trunc [[BROADCAST_SPLAT]] to @@ -84,8 +86,6 @@ define void @trunc_minimal_bitwidths_shufflevector (ptr %p, i32 %arg1, i64 %len) ; CHECK-NEXT: [[TMP6:%.*]] = xor [[WIDE_LOAD]], [[TMP4]] ; CHECK-NEXT: [[TMP7:%.*]] = mul [[TMP6]], [[WIDE_LOAD]] ; CHECK-NEXT: store [[TMP7]], ptr [[TMP5]], align 1 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll index 9a4f174f439af8780d4171c73c53ffa1634b2861..305fb7f8ab8d38938f5458479f3be1b2a86cd1f6 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll @@ -8,7 +8,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounwind uwtable { ; CHECK: digraph VPlan { -; CHECK-NEXT: graph [labelloc=t, fontsize=30; label="Vectorization Plan\nInitial VPlan for VF=\{4\},UF\>=1\nLive-in vp\<[[VEC_TC:%.+]]\> = vector-trip-count\nLive-in ir\<%n\> = original trip-count\n"] +; CHECK-NEXT: graph [labelloc=t, fontsize=30; label="Vectorization Plan\nInitial VPlan for VF=\{4\},UF\>=1\nLive-in vp\<[[VFxUF:%.+]]\> = VF * UF\nLive-in vp\<[[VEC_TC:%.+]]\> = vector-trip-count\nLive-in ir\<%n\> = original trip-count\n"] ; CHECK-NEXT: node [shape=rect, fontname=Courier, fontsize=30] ; CHECK-NEXT: edge [fontname=Courier, fontsize=30] ; CHECK-NEXT: compound=true @@ -33,7 +33,7 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: " WIDEN-CALL ir\<%call\> = call @llvm.sqrt.f32(ir\<%lv\>) (using vector intrinsic)\l" + ; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" + ; CHECK-NEXT: " WIDEN store ir\<%arrayidx2\>, ir\<%call\>\l" + -; CHECK-NEXT: " EMIT vp\<[[CAN_IV_NEXT]]\> = VF * UF + nuw vp\<[[CAN_IV]]\>\l" + +; CHECK-NEXT: " EMIT vp\<[[CAN_IV_NEXT]]\> = add nuw vp\<[[CAN_IV]]\>, vp\<[[VFxUF]]\>\l" + ; CHECK-NEXT: " EMIT branch-on-count vp\<[[CAN_IV_NEXT]]\>, vp\<[[VEC_TC]]\>\l" + ; CHECK-NEXT: "No successors\l" ; CHECK-NEXT: ] diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll index 70ede8dc12b9c7857491aedc941b0a9dfe8ef2ea..8d50993faf9e22e74f75ac148b2b6acbfc4ff029 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll @@ -4,6 +4,7 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) { ; CHECK-LABEL: LV: Checking a loop in 'iv_no_binary_op_in_descriptor' ; CHECK: VPlan 'Initial VPlan for VF={8},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: @@ -17,7 +18,7 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) { ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%dst>, vp<[[STEPS:%.+]]> ; CHECK-NEXT: WIDEN store ir<%gep>, ir<%iv> -; CHECK-NEXT: EMIT vp<[[CAN_INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_INC]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll index f129cf3e2a1aa5ec97e8aadb48eeb67c956500be..13f444eaaf76a595cec6f6069b98d525a86b4a4c 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -9,6 +9,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounwind uwtable { ; CHECK-LABEL: Checking a loop in 'print_call_and_memory' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -24,7 +25,7 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>) ; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx2>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -56,6 +57,7 @@ for.end: ; preds = %for.body, %entry define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x, ptr %z) nounwind uwtable { ; CHECK-LABEL: Checking a loop in 'print_widen_gep_and_select' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -74,7 +76,7 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x, ; CHECK-NEXT: WIDEN ir<%add> = fadd ir<%lv>, ir<%sel> ; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx2>, ir<%add> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -108,6 +110,7 @@ for.end: ; preds = %for.body, %entry define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-LABEL: Checking a loop in 'print_reduction' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -122,7 +125,7 @@ define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> ; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -154,6 +157,7 @@ for.end: ; preds = %for.body, %entry define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr noalias %dst) { ; CHECK-LABEL: Checking a loop in 'print_reduction_with_invariant_store' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -168,7 +172,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[IV]]> ; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> ; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) (with final reduction value stored in invariant address sank outside of loop) -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -199,6 +203,7 @@ for.end: ; preds = %for.body, %entry define void @print_replicate_predicated_phi(i64 %n, ptr %x) { ; CHECK-LABEL: Checking a loop in 'print_replicate_predicated_phi' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count ; CHECK-EMPTY: @@ -237,7 +242,7 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) { ; CHECK-NEXT: BLEND ir<%d> = ir<0>/vp<[[NOT]]> vp<[[PRED]]>/ir<%cmp> ; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%x>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%idx>, ir<%d> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -277,6 +282,7 @@ for.end: ; preds = %for.inc define void @print_interleave_groups(i32 %C, i32 %D) { ; CHECK-LABEL: Checking a loop in 'print_interleave_groups' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<256> = original trip-count ; CHECK-EMPTY: @@ -301,7 +307,7 @@ define void @print_interleave_groups(i32 %C, i32 %D) { ; CHECK-NEXT: store ir<1> to index 1 ; CHECK-NEXT: store ir<2> to index 2 ; CHECK-NEXT: store ir<%AB.3> to index 3 -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -345,6 +351,7 @@ for.end: define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-LABEL: Checking a loop in 'print_fmuladd_strict' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -362,7 +369,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: WIDEN ir<%l.b> = load ir<%arrayidx2> ; CHECK-NEXT: EMIT vp<[[FMUL:%.+]]> = fmul nnan ninf nsz ir<%l.a>, ir<%l.b> ; CHECK-NEXT: REDUCE ir<[[MULADD:%.+]]> = ir<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>) -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -396,6 +403,7 @@ for.end: define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !dbg !5 { ; CHECK-LABEL: Checking a loop in 'debug_loc_vpinstruction' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<128> = original trip-count ; CHECK-EMPTY: @@ -436,7 +444,7 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db ; CHECK-NEXT: EMIT vp<[[SEL2:%.+]]> = select vp<[[NOT1]]>, vp<[[NOT2]]>, ir ; CHECK-NEXT: BLEND ir<%ysd.0> = vp<[[PHI]]>/vp<[[OR1]]> ir<%psd>/vp<[[SEL2]]> ; CHECK-NEXT: WIDEN store ir<%isd>, ir<%ysd.0> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT:} @@ -482,6 +490,7 @@ declare float @llvm.fmuladd.f32(float, float, float) define void @print_expand_scev(i64 %y, ptr %ptr) { ; CHECK-LABEL: Checking a loop in 'print_expand_scev' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count ; CHECK-EMPTY: @@ -504,7 +513,7 @@ define void @print_expand_scev(i64 %y, ptr %ptr) { ; CHECK-NEXT: WIDEN ir<%v3> = add nuw ir<%v2>, ir<1> ; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> ; CHECK-NEXT: REPLICATE store ir<%v3>, ir<%gep> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -537,6 +546,7 @@ loop.exit: define i32 @print_exit_value(ptr %ptr, i32 %off) { ; CHECK-LABEL: Checking a loop in 'print_exit_value' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: @@ -551,7 +561,7 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) { ; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN ir<%add> = add ir<%iv>, ir<%off> ; CHECK-NEXT: WIDEN store ir<%gep>, ir<0> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -583,6 +593,7 @@ exit: define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr %z) { ; CHECK-LABEL: Checking a loop in 'print_fast_math_flags' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -600,7 +611,7 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr % ; CHECK-NEXT: WIDEN ir<%div> = fdiv reassoc nsz contract ir<%mul>, ir<2.000000e+00> ; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%gep.x>, ir<%div> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -633,6 +644,7 @@ exit: define void @print_exact_flags(i64 %n, ptr noalias %x) { ; CHECK-LABEL: Checking a loop in 'print_exact_flags' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -649,7 +661,7 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: WIDEN ir<%div.2> = udiv ir<%lv>, ir<60> ; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%div.1>, ir<%div.2> ; CHECK-NEXT: WIDEN store ir<%gep.x>, ir<%add> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -681,6 +693,7 @@ exit: define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) { ; CHECK-LABEL: Checking a loop in 'print_call_flags' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -719,7 +732,7 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) { ; CHECK-NEXT: BLEND ir<%st.value> = ir<%ld.value>/vp<[[NOT_COND]]> ir<%fadd>/ir<%ifcond> ; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr inbounds ir<%dest>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%st.addr>, ir<%st.value> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -761,6 +774,7 @@ end: define void @print_disjoint_flags(i64 %n, ptr noalias %x) { ; CHECK-LABEL: Checking a loop in 'print_disjoint_flags' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -777,7 +791,7 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: WIDEN ir<%or.2> = or ir<%lv>, ir<3> ; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%or.1>, ir<%or.2> ; CHECK-NEXT: WIDEN store ir<%gep.x>, ir<%add> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -809,23 +823,23 @@ exit: define void @zext_nneg(ptr noalias %p, ptr noalias %p1) { ; CHECK-LABEL: LV: Checking a loop in 'zext_nneg' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<%0> = vector-trip-count -; CHECK-NEXT: Live-in ir<0> = original trip-count +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION ir<0>, vp<%8> -; CHECK-NEXT: vp<%2> = DERIVED-IV ir<0> + vp<%1> * ir<1> (truncated to i32) -; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1> -; CHECK-NEXT: CLONE ir<%zext> = zext nneg vp<%3> -; CHECK-NEXT: CLONE ir<%idx2> = getelementptr ir<%p>, ir<%zext> -; CHECK-NEXT: WIDEN ir<%1> = load ir<%idx2> -; CHECK-NEXT: REPLICATE store ir<%1>, ir<%p1> -; CHECK-NEXT: EMIT vp<%8> = VF * UF + nuw vp<%1> -; CHECK-NEXT: EMIT branch-on-count vp<%8>, vp<%0> +; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> +; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%p>, vp<[[STEPS]]> +; CHECK-NEXT: WIDEN ir<%l> = load ir<%idx> +; CHECK-NEXT: WIDEN-CAST ir<%zext> = zext nneg ir<%l> +; CHECK-NEXT: REPLICATE store ir<%zext>, ir<%p1> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -834,13 +848,12 @@ entry: body: %iv = phi i64 [ %next, %body ], [ 0, %entry ] - %0 = trunc i64 %iv to i32 - %zext = zext nneg i32 %0 to i64 - %idx2 = getelementptr double, ptr %p, i64 %zext - %1 = load double, ptr %idx2, align 8 - store double %1, ptr %p1, align 8 + %idx = getelementptr i32, ptr %p, i64 %iv + %l = load i32, ptr %idx, align 8 + %zext = zext nneg i32 %l to i64 + store i64 %zext, ptr %p1, align 8 %next = add i64 %iv, 1 - %cmp = icmp eq i64 %next, 0 + %cmp = icmp eq i64 %next, 1000 br i1 %cmp, label %exit, label %body exit: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll index 0dac63d54557d7ab4926b7c1037276dca455998b..8c18c107b39fc10962bb5e0497a9a91642577ff6 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll @@ -8,6 +8,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-LABEL: sink_with_sideeffects ; CHECK: VPlan 'Initial VPlan for VF={1},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: ir<0> = original trip-count ; CHECK-EMPTY: @@ -37,7 +38,7 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-NEXT: } ; CHECK: if.then.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll index afc46de9a5a168e9337e3bdfbadb8fa4aa67925e..f2e6dd43a2ec9484d8ecfd647b8d4e00c370c01f 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -11,6 +11,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; CHECK-LABEL: LV: Checking a loop in 'sink1' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -50,7 +51,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; CHECK-NEXT: } ; CHECK: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -79,6 +80,7 @@ exit: ; CHECK-LABEL: LV: Checking a loop in 'sink2' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -133,7 +135,7 @@ exit: ; CHECK-NEXT: } ; CHECK: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -162,6 +164,7 @@ exit: ; CHECK-LABEL: LV: Checking a loop in 'sink3' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -216,7 +219,7 @@ exit: ; CHECK-NEXT: } ; CHECK: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -247,6 +250,7 @@ exit: define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) { ; CHECK-LABEL: LV: Checking a loop in 'uniform_gep' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<11> = original trip-count @@ -285,7 +289,7 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) { ; CHECK-NEXT: Successor(s): loop.then.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.then.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -317,6 +321,7 @@ exit: define void @pred_cfg1(i32 %k, i32 %j) { ; CHECK-LABEL: LV: Checking a loop in 'pred_cfg1' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -378,7 +383,7 @@ define void @pred_cfg1(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): next.0.1 ; CHECK-EMPTY: ; CHECK-NEXT: next.0.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -416,6 +421,7 @@ exit: define void @pred_cfg2(i32 %k, i32 %j) { ; CHECK-LABEL: LV: Checking a loop in 'pred_cfg2' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -479,7 +485,7 @@ define void @pred_cfg2(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): then.1.1 ; CHECK-EMPTY: ; CHECK-NEXT: then.1.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -524,6 +530,7 @@ exit: define void @pred_cfg3(i32 %k, i32 %j) { ; CHECK-LABEL: LV: Checking a loop in 'pred_cfg3' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -588,7 +595,7 @@ define void @pred_cfg3(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): then.1.2 ; CHECK-EMPTY: ; CHECK-NEXT: then.1.2: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -632,6 +639,7 @@ exit: define void @merge_3_replicate_region(i32 %k, i32 %j) { ; CHECK-LABEL: LV: Checking a loop in 'merge_3_replicate_region' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -695,7 +703,7 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): then.0.4 ; CHECK-EMPTY: ; CHECK-NEXT: then.0.4: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -736,6 +744,7 @@ exit: define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) { ; CHECK-LABEL: LV: Checking a loop in 'update_2_uses_in_same_recipe_in_merged_block' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -775,7 +784,7 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) { ; CHECK-NEXT: Successor(s): loop.2 ; CHECK-EMPTY: ; CHECK-NEXT: loop.2: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -802,6 +811,7 @@ exit: define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) { ; CHECK-LABEL: LV: Checking a loop in 'recipe_in_merge_candidate_used_by_first_order_recurrence' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -859,7 +869,7 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) { ; CHECK-NEXT: Successor(s): loop.2 ; CHECK-EMPTY: ; CHECK-NEXT: loop.2: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -887,6 +897,7 @@ exit: define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) { ; CHECK-LABEL: LV: Checking a loop in 'update_multiple_users' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<999> = original trip-count ; CHECK-EMPTY: @@ -918,7 +929,7 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) { ; CHECK-NEXT: Successor(s): loop.then.1 ; CHECK-EMPTY: ; CHECK-NEXT: loop.then.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -951,6 +962,7 @@ exit: define void @sinking_requires_duplication(ptr %addr) { ; CHECK-LABEL: LV: Checking a loop in 'sinking_requires_duplication' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<201> = original trip-count ; CHECK-EMPTY: @@ -983,7 +995,7 @@ define void @sinking_requires_duplication(ptr %addr) { ; CHECK-NEXT: Successor(s): then.0 ; CHECK-EMPTY: ; CHECK-NEXT: then.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -1019,6 +1031,7 @@ exit: define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr noalias %dst) optsize { ; CHECK-LABEL: LV: Checking a loop in 'merge_with_dead_gep_between_regions' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<%n> = original trip-count @@ -1054,7 +1067,7 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n ; CHECK-NEXT: Successor(s): loop.1 ; CHECK-EMPTY: ; CHECK-NEXT: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -1085,6 +1098,7 @@ exit: define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-LABEL: LV: Checking a loop in 'ptr_induction_remove_dead_recipe' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count ; CHECK-EMPTY: @@ -1121,7 +1135,7 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-NEXT: Successor(s): if.then.0 ; CHECK-EMPTY: ; CHECK-NEXT: if.then.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/MemCpyOpt/pr75010.ll b/llvm/test/Transforms/MemCpyOpt/pr75010.ll new file mode 100644 index 0000000000000000000000000000000000000000..c9af7543b42e64b1824251ff1ad8102420703f6a --- /dev/null +++ b/llvm/test/Transforms/MemCpyOpt/pr75010.ll @@ -0,0 +1,19 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -passes=memcpyopt < %s | FileCheck %s + +define void @f(ptr nocapture noundef writeonly %r, %x) { +; CHECK-LABEL: @f( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[X_ADDR:%.*]] = alloca , align 8 +; CHECK-NEXT: store [[X:%.*]], ptr [[X_ADDR]], align 8 +; CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr [[X_ADDR]], align 8 +; CHECK-NEXT: store i64 [[TMP0]], ptr [[R:%.*]], align 1 +; CHECK-NEXT: ret void +; +entry: + %x.addr = alloca , align 8 + store %x, ptr %x.addr, align 8 + %0 = load i64, ptr %x.addr, align 8 + store i64 %0, ptr %r, align 1 + ret void +} diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll b/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll index e5582548447aed0cbf9bc29ab032859401783a17..2510aeaebcf0cfc09664c700373b1ee58f23f637 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll @@ -64,31 +64,31 @@ define void @vdiv(ptr %x, ptr %y, double %a, i32 %N) #0 { ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY_PREHEADER9]] ; CHECK: for.body.preheader9: ; CHECK-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] -; CHECK-NEXT: [[TMP18:%.*]] = xor i64 [[INDVARS_IV_PH]], -1 -; CHECK-NEXT: [[TMP19:%.*]] = add nsw i64 [[TMP18]], [[WIDE_TRIP_COUNT]] ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 7 ; CHECK-NEXT: [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 0 ; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label [[FOR_BODY_PROL_LOOPEXIT:%.*]], label [[FOR_BODY_PROL_PREHEADER:%.*]] ; CHECK: for.body.prol.preheader: -; CHECK-NEXT: [[TMP20:%.*]] = fdiv fast double 1.000000e+00, [[A]] +; CHECK-NEXT: [[TMP18:%.*]] = fdiv fast double 1.000000e+00, [[A]] ; CHECK-NEXT: br label [[FOR_BODY_PROL:%.*]] ; CHECK: for.body.prol: ; CHECK-NEXT: [[INDVARS_IV_PROL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_PROL:%.*]], [[FOR_BODY_PROL]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PROL_PREHEADER]] ] ; CHECK-NEXT: [[PROL_ITER:%.*]] = phi i64 [ [[PROL_ITER_NEXT:%.*]], [[FOR_BODY_PROL]] ], [ 0, [[FOR_BODY_PROL_PREHEADER]] ] ; CHECK-NEXT: [[ARRAYIDX_PROL:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_PROL]] ; CHECK-NEXT: [[T0_PROL:%.*]] = load double, ptr [[ARRAYIDX_PROL]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP21:%.*]] = fmul fast double [[T0_PROL]], [[TMP20]] +; CHECK-NEXT: [[TMP19:%.*]] = fmul fast double [[T0_PROL]], [[TMP18]] ; CHECK-NEXT: [[ARRAYIDX2_PROL:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_PROL]] -; CHECK-NEXT: store double [[TMP21]], ptr [[ARRAYIDX2_PROL]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP19]], ptr [[ARRAYIDX2_PROL]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_PROL]] = add nuw nsw i64 [[INDVARS_IV_PROL]], 1 ; CHECK-NEXT: [[PROL_ITER_NEXT]] = add i64 [[PROL_ITER]], 1 ; CHECK-NEXT: [[PROL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[PROL_ITER_NEXT]], [[XTRAITER]] ; CHECK-NEXT: br i1 [[PROL_ITER_CMP_NOT]], label [[FOR_BODY_PROL_LOOPEXIT]], label [[FOR_BODY_PROL]], !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: for.body.prol.loopexit: ; CHECK-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER9]] ], [ [[INDVARS_IV_NEXT_PROL]], [[FOR_BODY_PROL]] ] -; CHECK-NEXT: [[TMP22:%.*]] = icmp ult i64 [[TMP19]], 7 -; CHECK-NEXT: br i1 [[TMP22]], label [[FOR_END]], label [[FOR_BODY_PREHEADER9_NEW:%.*]] +; CHECK-NEXT: [[TMP20:%.*]] = sub nsw i64 [[INDVARS_IV_PH]], [[WIDE_TRIP_COUNT]] +; CHECK-NEXT: [[TMP21:%.*]] = icmp ugt i64 [[TMP20]], -8 +; CHECK-NEXT: br i1 [[TMP21]], label [[FOR_END]], label [[FOR_BODY_PREHEADER9_NEW:%.*]] ; CHECK: for.body.preheader9.new: +; CHECK-NEXT: [[TMP22:%.*]] = fdiv fast double 1.000000e+00, [[A]] ; CHECK-NEXT: [[TMP23:%.*]] = fdiv fast double 1.000000e+00, [[A]] ; CHECK-NEXT: [[TMP24:%.*]] = fdiv fast double 1.000000e+00, [[A]] ; CHECK-NEXT: [[TMP25:%.*]] = fdiv fast double 1.000000e+00, [[A]] @@ -96,57 +96,56 @@ define void @vdiv(ptr %x, ptr %y, double %a, i32 %N) #0 { ; CHECK-NEXT: [[TMP27:%.*]] = fdiv fast double 1.000000e+00, [[A]] ; CHECK-NEXT: [[TMP28:%.*]] = fdiv fast double 1.000000e+00, [[A]] ; CHECK-NEXT: [[TMP29:%.*]] = fdiv fast double 1.000000e+00, [[A]] -; CHECK-NEXT: [[TMP30:%.*]] = fdiv fast double 1.000000e+00, [[A]] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_UNR]], [[FOR_BODY_PREHEADER9_NEW]] ], [ [[INDVARS_IV_NEXT_7:%.*]], [[FOR_BODY]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV]] ; CHECK-NEXT: [[T0:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP31:%.*]] = fmul fast double [[T0]], [[TMP23]] +; CHECK-NEXT: [[TMP30:%.*]] = fmul fast double [[T0]], [[TMP22]] ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV]] -; CHECK-NEXT: store double [[TMP31]], ptr [[ARRAYIDX2]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP30]], ptr [[ARRAYIDX2]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT]] ; CHECK-NEXT: [[T0_1:%.*]] = load double, ptr [[ARRAYIDX_1]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP32:%.*]] = fmul fast double [[T0_1]], [[TMP24]] +; CHECK-NEXT: [[TMP31:%.*]] = fmul fast double [[T0_1]], [[TMP23]] ; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT]] -; CHECK-NEXT: store double [[TMP32]], ptr [[ARRAYIDX2_1]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP31]], ptr [[ARRAYIDX2_1]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2 ; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_1]] ; CHECK-NEXT: [[T0_2:%.*]] = load double, ptr [[ARRAYIDX_2]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP33:%.*]] = fmul fast double [[T0_2]], [[TMP25]] +; CHECK-NEXT: [[TMP32:%.*]] = fmul fast double [[T0_2]], [[TMP24]] ; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_1]] -; CHECK-NEXT: store double [[TMP33]], ptr [[ARRAYIDX2_2]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP32]], ptr [[ARRAYIDX2_2]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_2:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 3 ; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_2]] ; CHECK-NEXT: [[T0_3:%.*]] = load double, ptr [[ARRAYIDX_3]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP34:%.*]] = fmul fast double [[T0_3]], [[TMP26]] +; CHECK-NEXT: [[TMP33:%.*]] = fmul fast double [[T0_3]], [[TMP25]] ; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_2]] -; CHECK-NEXT: store double [[TMP34]], ptr [[ARRAYIDX2_3]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP33]], ptr [[ARRAYIDX2_3]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_3:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4 ; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_3]] ; CHECK-NEXT: [[T0_4:%.*]] = load double, ptr [[ARRAYIDX_4]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP35:%.*]] = fmul fast double [[T0_4]], [[TMP27]] +; CHECK-NEXT: [[TMP34:%.*]] = fmul fast double [[T0_4]], [[TMP26]] ; CHECK-NEXT: [[ARRAYIDX2_4:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_3]] -; CHECK-NEXT: store double [[TMP35]], ptr [[ARRAYIDX2_4]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP34]], ptr [[ARRAYIDX2_4]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_4:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 5 ; CHECK-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_4]] ; CHECK-NEXT: [[T0_5:%.*]] = load double, ptr [[ARRAYIDX_5]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP36:%.*]] = fmul fast double [[T0_5]], [[TMP28]] +; CHECK-NEXT: [[TMP35:%.*]] = fmul fast double [[T0_5]], [[TMP27]] ; CHECK-NEXT: [[ARRAYIDX2_5:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_4]] -; CHECK-NEXT: store double [[TMP36]], ptr [[ARRAYIDX2_5]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP35]], ptr [[ARRAYIDX2_5]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_5:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 6 ; CHECK-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_5]] ; CHECK-NEXT: [[T0_6:%.*]] = load double, ptr [[ARRAYIDX_6]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP37:%.*]] = fmul fast double [[T0_6]], [[TMP29]] +; CHECK-NEXT: [[TMP36:%.*]] = fmul fast double [[T0_6]], [[TMP28]] ; CHECK-NEXT: [[ARRAYIDX2_6:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_5]] -; CHECK-NEXT: store double [[TMP37]], ptr [[ARRAYIDX2_6]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP36]], ptr [[ARRAYIDX2_6]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_6:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 7 ; CHECK-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_6]] ; CHECK-NEXT: [[T0_7:%.*]] = load double, ptr [[ARRAYIDX_7]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP38:%.*]] = fmul fast double [[T0_7]], [[TMP30]] +; CHECK-NEXT: [[TMP37:%.*]] = fmul fast double [[T0_7]], [[TMP29]] ; CHECK-NEXT: [[ARRAYIDX2_7:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_6]] -; CHECK-NEXT: store double [[TMP38]], ptr [[ARRAYIDX2_7]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP37]], ptr [[ARRAYIDX2_7]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_7]] = add nuw nsw i64 [[INDVARS_IV]], 8 ; CHECK-NEXT: [[EXITCOND_NOT_7:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_7]], [[WIDE_TRIP_COUNT]] ; CHECK-NEXT: br i1 [[EXITCOND_NOT_7]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] diff --git a/llvm/test/Transforms/Reassociate/local-cse.ll b/llvm/test/Transforms/Reassociate/local-cse.ll index 1609cb1b36fd93e6dbc68069f2bb7e883ac0c5ad..4d0467e263f5538e8c6770e3facf223fe5723bea 100644 --- a/llvm/test/Transforms/Reassociate/local-cse.ll +++ b/llvm/test/Transforms/Reassociate/local-cse.ll @@ -26,16 +26,16 @@ define void @chain_spanning_several_blocks(i64 %inv1, i64 %inv2, i64 %inv3, i64 ; LOCAL_CSE-LABEL: define void @chain_spanning_several_blocks ; LOCAL_CSE-SAME: (i64 [[INV1:%.*]], i64 [[INV2:%.*]], i64 [[INV3:%.*]], i64 [[INV4:%.*]], i64 [[INV5:%.*]]) { ; LOCAL_CSE-NEXT: bb1: -; LOCAL_CSE-NEXT: [[CHAIN_A0:%.*]] = add i64 [[INV2]], [[INV1]] +; LOCAL_CSE-NEXT: [[CHAIN_A0:%.*]] = add nuw i64 [[INV2]], [[INV1]] ; LOCAL_CSE-NEXT: br label [[BB2:%.*]] ; LOCAL_CSE: bb2: ; LOCAL_CSE-NEXT: [[VAL_BB2:%.*]] = call i64 @get_val() -; LOCAL_CSE-NEXT: [[CHAIN_A1:%.*]] = add i64 [[CHAIN_A0]], [[INV4]] -; LOCAL_CSE-NEXT: [[CHAIN_A2:%.*]] = add i64 [[CHAIN_A1]], [[VAL_BB2]] -; LOCAL_CSE-NEXT: [[CHAIN_B1:%.*]] = add i64 [[CHAIN_A0]], [[INV5]] -; LOCAL_CSE-NEXT: [[CHAIN_B2:%.*]] = add i64 [[CHAIN_B1]], [[VAL_BB2]] -; LOCAL_CSE-NEXT: [[CHAIN_C0:%.*]] = add i64 [[INV3]], [[INV1]] -; LOCAL_CSE-NEXT: [[CHAIN_C1:%.*]] = add i64 [[CHAIN_C0]], [[VAL_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_A1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV4]] +; LOCAL_CSE-NEXT: [[CHAIN_A2:%.*]] = add nuw i64 [[CHAIN_A1]], [[VAL_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_B1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV5]] +; LOCAL_CSE-NEXT: [[CHAIN_B2:%.*]] = add nuw i64 [[CHAIN_B1]], [[VAL_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_C0:%.*]] = add nuw i64 [[INV3]], [[INV1]] +; LOCAL_CSE-NEXT: [[CHAIN_C1:%.*]] = add nuw i64 [[CHAIN_C0]], [[VAL_BB2]] ; LOCAL_CSE-NEXT: call void @keep_alive(i64 [[CHAIN_A2]]) ; LOCAL_CSE-NEXT: call void @keep_alive(i64 [[CHAIN_B2]]) ; LOCAL_CSE-NEXT: call void @keep_alive(i64 [[CHAIN_C1]]) @@ -47,11 +47,11 @@ define void @chain_spanning_several_blocks(i64 %inv1, i64 %inv2, i64 %inv3, i64 ; CSE-NEXT: br label [[BB2:%.*]] ; CSE: bb2: ; CSE-NEXT: [[VAL_BB2:%.*]] = call i64 @get_val() -; CSE-NEXT: [[CHAIN_A0:%.*]] = add i64 [[VAL_BB2]], [[INV1]] -; CSE-NEXT: [[CHAIN_A1:%.*]] = add i64 [[CHAIN_A0]], [[INV2]] +; CSE-NEXT: [[CHAIN_A0:%.*]] = add nuw i64 [[VAL_BB2]], [[INV1]] +; CSE-NEXT: [[CHAIN_A1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV2]] ; CSE-NEXT: [[CHAIN_A2:%.*]] = add nuw nsw i64 [[CHAIN_A1]], [[INV4]] ; CSE-NEXT: [[CHAIN_B2:%.*]] = add nuw nsw i64 [[CHAIN_A1]], [[INV5]] -; CSE-NEXT: [[CHAIN_C1:%.*]] = add i64 [[CHAIN_A0]], [[INV3]] +; CSE-NEXT: [[CHAIN_C1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV3]] ; CSE-NEXT: call void @keep_alive(i64 [[CHAIN_A2]]) ; CSE-NEXT: call void @keep_alive(i64 [[CHAIN_B2]]) ; CSE-NEXT: call void @keep_alive(i64 [[CHAIN_C1]]) @@ -90,19 +90,19 @@ define void @chain_spanning_several_blocks_no_entry_anchor() { ; LOCAL_CSE-NEXT: br label [[BB1:%.*]] ; LOCAL_CSE: bb1: ; LOCAL_CSE-NEXT: [[INV1_BB1:%.*]] = call i64 @get_val() -; LOCAL_CSE-NEXT: [[CHAIN_A0:%.*]] = add i64 [[INV1_BB1]], [[INV2_BB0]] +; LOCAL_CSE-NEXT: [[CHAIN_A0:%.*]] = add nuw i64 [[INV1_BB1]], [[INV2_BB0]] ; LOCAL_CSE-NEXT: br label [[BB2:%.*]] ; LOCAL_CSE: bb2: ; LOCAL_CSE-NEXT: [[INV3_BB2:%.*]] = call i64 @get_val() ; LOCAL_CSE-NEXT: [[INV4_BB2:%.*]] = call i64 @get_val() ; LOCAL_CSE-NEXT: [[INV5_BB2:%.*]] = call i64 @get_val() ; LOCAL_CSE-NEXT: [[VAL_BB2:%.*]] = call i64 @get_val() -; LOCAL_CSE-NEXT: [[CHAIN_A1:%.*]] = add i64 [[CHAIN_A0]], [[INV4_BB2]] -; LOCAL_CSE-NEXT: [[CHAIN_A2:%.*]] = add i64 [[CHAIN_A1]], [[VAL_BB2]] -; LOCAL_CSE-NEXT: [[CHAIN_B1:%.*]] = add i64 [[CHAIN_A0]], [[INV5_BB2]] -; LOCAL_CSE-NEXT: [[CHAIN_B2:%.*]] = add i64 [[CHAIN_B1]], [[VAL_BB2]] -; LOCAL_CSE-NEXT: [[CHAIN_C0:%.*]] = add i64 [[VAL_BB2]], [[INV1_BB1]] -; LOCAL_CSE-NEXT: [[CHAIN_C1:%.*]] = add i64 [[CHAIN_C0]], [[INV3_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_A1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV4_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_A2:%.*]] = add nuw i64 [[CHAIN_A1]], [[VAL_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_B1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV5_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_B2:%.*]] = add nuw i64 [[CHAIN_B1]], [[VAL_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_C0:%.*]] = add nuw i64 [[VAL_BB2]], [[INV1_BB1]] +; LOCAL_CSE-NEXT: [[CHAIN_C1:%.*]] = add nuw i64 [[CHAIN_C0]], [[INV3_BB2]] ; LOCAL_CSE-NEXT: call void @keep_alive(i64 [[CHAIN_A2]]) ; LOCAL_CSE-NEXT: call void @keep_alive(i64 [[CHAIN_B2]]) ; LOCAL_CSE-NEXT: call void @keep_alive(i64 [[CHAIN_C1]]) @@ -120,11 +120,11 @@ define void @chain_spanning_several_blocks_no_entry_anchor() { ; CSE-NEXT: [[INV4_BB2:%.*]] = call i64 @get_val() ; CSE-NEXT: [[INV5_BB2:%.*]] = call i64 @get_val() ; CSE-NEXT: [[VAL_BB2:%.*]] = call i64 @get_val() -; CSE-NEXT: [[CHAIN_A0:%.*]] = add i64 [[VAL_BB2]], [[INV1_BB1]] -; CSE-NEXT: [[CHAIN_A1:%.*]] = add i64 [[CHAIN_A0]], [[INV2_BB0]] +; CSE-NEXT: [[CHAIN_A0:%.*]] = add nuw i64 [[VAL_BB2]], [[INV1_BB1]] +; CSE-NEXT: [[CHAIN_A1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV2_BB0]] ; CSE-NEXT: [[CHAIN_A2:%.*]] = add nuw nsw i64 [[CHAIN_A1]], [[INV4_BB2]] ; CSE-NEXT: [[CHAIN_B2:%.*]] = add nuw nsw i64 [[CHAIN_A1]], [[INV5_BB2]] -; CSE-NEXT: [[CHAIN_C1:%.*]] = add i64 [[CHAIN_A0]], [[INV3_BB2]] +; CSE-NEXT: [[CHAIN_C1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV3_BB2]] ; CSE-NEXT: call void @keep_alive(i64 [[CHAIN_A2]]) ; CSE-NEXT: call void @keep_alive(i64 [[CHAIN_B2]]) ; CSE-NEXT: call void @keep_alive(i64 [[CHAIN_C1]]) diff --git a/llvm/test/Transforms/Reassociate/reassoc-mul-nuw.ll b/llvm/test/Transforms/Reassociate/reassoc-mul-nuw.ll new file mode 100644 index 0000000000000000000000000000000000000000..682fad8d222b7cab5475b2c6d62761c3b1e1b1be --- /dev/null +++ b/llvm/test/Transforms/Reassociate/reassoc-mul-nuw.ll @@ -0,0 +1,34 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes=reassociate -S | FileCheck %s + +; We cannot preserve nuw flags for mul +define i4 @nuw_preserve_negative(i4 %a, i4 %b, i4 %c) { +; CHECK-LABEL: define i4 @nuw_preserve_negative( +; CHECK-SAME: i4 [[A:%.*]], i4 [[B:%.*]], i4 [[C:%.*]]) { +; CHECK-NEXT: [[V0:%.*]] = mul i4 [[B]], [[A]] +; CHECK-NEXT: [[V1:%.*]] = mul i4 [[V0]], [[C]] +; CHECK-NEXT: ret i4 [[V1]] +; + %v0 = mul nuw i4 %a, %c + %v1 = mul nuw i4 %v0, %b + ret i4 %v1 +} + +; TODO: we can add nuw flags if we know all operands are non-zero. +define i4 @nuw_preserve_non_zero(i4 %a, i4 %b, i4 %c) { +; CHECK-LABEL: define i4 @nuw_preserve_non_zero( +; CHECK-SAME: i4 [[A:%.*]], i4 [[B:%.*]], i4 [[C:%.*]]) { +; CHECK-NEXT: [[A0:%.*]] = add nuw i4 [[A]], 1 +; CHECK-NEXT: [[B0:%.*]] = add nuw i4 [[B]], 1 +; CHECK-NEXT: [[C0:%.*]] = add nuw i4 [[C]], 1 +; CHECK-NEXT: [[V0:%.*]] = mul i4 [[B0]], [[A0]] +; CHECK-NEXT: [[V1:%.*]] = mul i4 [[V0]], [[C0]] +; CHECK-NEXT: ret i4 [[V1]] +; + %a0 = add nuw i4 %a, 1 + %b0 = add nuw i4 %b, 1 + %c0 = add nuw i4 %c, 1 + %v0 = mul nuw i4 %a0, %c0 + %v1 = mul nuw i4 %v0, %b0 + ret i4 %v1 +} diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll new file mode 100644 index 0000000000000000000000000000000000000000..8e0f382222241f2b120645715b6585e2be23abc1 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll @@ -0,0 +1,56 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S --passes=slp-vectorizer -mtriple=riscv64-unknown-linux -pass-remarks-output=%t -mattr=+v -slp-threshold=-10 < %s | FileCheck %s +; RUN: FileCheck %s --check-prefix=YAML < %t + +; YAML-LABEL: --- !Passed +; YAML-NEXT: Pass: slp-vectorizer +; YAML-NEXT: Name: StoresVectorized +; YAML-NEXT: Function: test +; YAML-NEXT: Args: +; YAML-NEXT: - String: 'Stores SLP vectorized with cost ' +; YAML-NEXT: - Cost: '9' +; YAML-NEXT: - String: ' and with tree size ' +; YAML-NEXT: - TreeSize: '7' + +define void @test() { +; CHECK-LABEL: define void @test( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr null, align 4 +; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr null, align 4 +; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr null, align 4 +; CHECK-NEXT: [[V9IDX:%.*]] = getelementptr i8, ptr null, i32 4 +; CHECK-NEXT: [[V14IDX:%.*]] = getelementptr i8, ptr null, i32 8 +; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x float> , float [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[TMP2]], i32 1 +; CHECK-NEXT: [[TMP6:%.*]] = fcmp ogt <2 x float> [[TMP3]], [[TMP5]] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP6]], i32 0 +; CHECK-NEXT: [[V0_0:%.*]] = select i1 [[TMP7]], float [[TMP0]], float 0.000000e+00 +; CHECK-NEXT: [[TMP8:%.*]] = select <2 x i1> [[TMP6]], <2 x float> [[TMP3]], <2 x float> zeroinitializer +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP6]], i32 1 +; CHECK-NEXT: [[V9_0:%.*]] = select i1 [[TMP9]], float [[TMP2]], float 0.000000e+00 +; CHECK-NEXT: store float [[V0_0]], ptr null, align 4 +; CHECK-NEXT: store float [[V9_0]], ptr [[V9IDX]], align 4 +; CHECK-NEXT: store <2 x float> [[TMP8]], ptr [[V14IDX]], align 4 +; CHECK-NEXT: ret void +; +entry: + %0 = load float, ptr null, align 4 + %1 = load float, ptr null, align 4 + %2 = load float, ptr null, align 4 + %cmp.i = fcmp ogt float %1, %0 + %v14.0 = select i1 %cmp.i, float %1, float 0.000000e+00 + %v0.0 = select i1 %cmp.i, float %0, float 0.000000e+00 + %cmp4.i = fcmp ogt float 0.000000e+00, %2 + %v19.0 = select i1 %cmp4.i, float 0.000000e+00, float 0.000000e+00 + %v9.0 = select i1 %cmp4.i, float %2, float 0.000000e+00 + store float %v0.0, ptr null, align 4 + %v9idx = getelementptr i8, ptr null, i32 4 + store float %v9.0, ptr %v9idx, align 4 + %v14idx = getelementptr i8, ptr null, i32 8 + store float %v14.0, ptr %v14idx, align 4 + %v19idx = getelementptr i8, ptr null, i32 12 + store float %v19.0, ptr %v19idx, align 4 + ret void +} diff --git a/llvm/test/tools/llvm-exegesis/X86/latency/memory-annotations-unsupported.s b/llvm/test/tools/llvm-exegesis/X86/latency/memory-annotations-unsupported.s index 5d06a0ceff5c7f8e6987e1f58937861643bd1fe9..c483d28adf0e17c48695800fef43300c26a5f5d2 100644 --- a/llvm/test/tools/llvm-exegesis/X86/latency/memory-annotations-unsupported.s +++ b/llvm/test/tools/llvm-exegesis/X86/latency/memory-annotations-unsupported.s @@ -2,7 +2,7 @@ # RUN: not llvm-exegesis -mtriple=x86_64-unknown-unknown -snippets-file=%s -mode=latency 2>&1 | FileCheck %s -# CHECK: llvm-exegesis error: Memory annotations are only supported in subprocess execution mode +# CHECK: llvm-exegesis error: Memory and snippet address annotations are only supported in subprocess execution mode # LLVM-EXEGESIS-MEM-DEF test1 4096 ff diff --git a/llvm/test/tools/llvm-exegesis/X86/latency/snippet-address-annotations-unsupported.s b/llvm/test/tools/llvm-exegesis/X86/latency/snippet-address-annotations-unsupported.s new file mode 100644 index 0000000000000000000000000000000000000000..c38c41d2aed9be1e3823b305c00f9535908c4b78 --- /dev/null +++ b/llvm/test/tools/llvm-exegesis/X86/latency/snippet-address-annotations-unsupported.s @@ -0,0 +1,9 @@ +# REQUIRES: exegesis-can-measure-latency, x86_64-linux + +# RUN: not llvm-exegesis -mtriple=x86_64-unknown-unknown -snippets-file=%s -mode=latency 2>&1 | FileCheck %s + +# CHECK: llvm-exegesis error: Memory and snippet address annotations are only supported in subprocess execution mode + +# LLVM-EXEGESIS-SNIPPET-ADDRESS 10000 + +movq $0, %rax diff --git a/llvm/test/tools/llvm-exegesis/X86/latency/subprocess-address-annotation.s b/llvm/test/tools/llvm-exegesis/X86/latency/subprocess-address-annotation.s new file mode 100644 index 0000000000000000000000000000000000000000..1b104b30bb288c353b1f99dbaf64b018ee821c10 --- /dev/null +++ b/llvm/test/tools/llvm-exegesis/X86/latency/subprocess-address-annotation.s @@ -0,0 +1,26 @@ +# REQUIRES: exegesis-can-measure-latency, x86_64-linux + +# RUN: llvm-exegesis -mtriple=x86_64-unknown-unknown -mode=latency -snippets-file=%s -execution-mode=subprocess | FileCheck %s + +# Check that the code is loaded in at the expected address. + +# LLVM-EXEGESIS-SNIPPET-ADDRESS 20000 +# LLVM-EXEGESIS-DEFREG RAX 0 +# LLVM-EXEGESIS-DEFREG R14 127 +# LLVM-EXEGESIS-DEFREG R15 0 +# LLVM-EXEGESIS-DEFREG RDI 0 + +# Load the instruction pointer and round down to the nearest page as there +# will be some setup code loaded in before this part begins to execute. +lea 0(%rip), %rax +shrq $12, %rax +shlq $12, %rax + +cmpq $0x20000, %rax +cmovneq %r14, %r15 + +movq $60, %rax +movq %r15, %rdi +syscall + +# CHECK-NOT: error: 'Child benchmarking process exited with non-zero exit code: Child process returned with unknown exit code' diff --git a/llvm/test/tools/llvm-objcopy/regex-error.test b/llvm/test/tools/llvm-objcopy/regex-error.test new file mode 100644 index 0000000000000000000000000000000000000000..ca9bb2f3b4f9f836a6633499b8a6b7cd2f4a2550 --- /dev/null +++ b/llvm/test/tools/llvm-objcopy/regex-error.test @@ -0,0 +1,13 @@ +## Test if providing objcopy with an invalid regex generates an error. + +# RUN: yaml2obj %s -o %t + +# RUN: not llvm-objcopy --regex --strip-symbol='[^)\' %t /dev/null 2>&1 | FileCheck %s +# CHECK: error: cannot compile regular expression '[^)\' + +!ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_EXEC + Machine: EM_X86_64 diff --git a/llvm/test/tools/llvm-objdump/ELF/dynamic-section-machine-specific.test b/llvm/test/tools/llvm-objdump/ELF/dynamic-section-machine-specific.test index 20219dd4893b77cd2a9cc9a53833b1c73c9c0cd1..203c210eb46a3ee82ef1d83c1056ec7aaa2d2744 100644 --- a/llvm/test/tools/llvm-objdump/ELF/dynamic-section-machine-specific.test +++ b/llvm/test/tools/llvm-objdump/ELF/dynamic-section-machine-specific.test @@ -268,6 +268,9 @@ ProgramHeaders: # AARCH64: Dynamic Section: # AARCH64-NEXT: AARCH64_BTI_PLT 0x0000000000000001 # AARCH64-NEXT: AARCH64_PAC_PLT 0x0000000000000002 +# AARCH64-NEXT: AARCH64_AUTH_RELR 0x0000000000000003 +# AARCH64-NEXT: AARCH64_AUTH_RELRSZ 0x0000000000000004 +# AARCH64-NEXT: AARCH64_AUTH_RELRENT 0x0000000000000005 --- !ELF FileHeader: @@ -283,6 +286,12 @@ Sections: Value: 1 - Tag: DT_AARCH64_PAC_PLT Value: 2 + - Tag: DT_AARCH64_AUTH_RELR + Value: 3 + - Tag: DT_AARCH64_AUTH_RELRSZ + Value: 4 + - Tag: DT_AARCH64_AUTH_RELRENT + Value: 5 - Tag: DT_NULL Value: 0 ProgramHeaders: diff --git a/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s b/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s new file mode 100644 index 0000000000000000000000000000000000000000..f28d92eae8575432ed47d36e9e5228f43284becf --- /dev/null +++ b/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s @@ -0,0 +1,98 @@ +# RUN: rm -rf %t && split-file %s %t && cd %t + +# RUN: llvm-mc -filetype=obj -triple=aarch64-linux-gnu abi-tag.s -o tag.o +# RUN: llvm-mc -filetype=obj -triple=aarch64-linux-gnu abi-tag-short.s -o tag-short.o +# RUN: llvm-mc -filetype=obj -triple=aarch64-linux-gnu abi-tag-long.s -o tag-long.o + +# RUN: llvm-readelf --notes tag.o | FileCheck --check-prefix NORMAL %s +# RUN: llvm-readelf --notes tag-short.o | FileCheck --check-prefix SHORT %s +# RUN: llvm-readelf --notes tag-long.o | FileCheck --check-prefix LONG %s + +# NORMAL: AArch64 PAuth ABI tag: platform 0x2a, version 0x1 +# SHORT: AArch64 PAuth ABI tag: +# LONG: AArch64 PAuth ABI tag: platform 0x2a, version 0x1, additional info 0xEFCDAB8967452301 + +# RUN: llvm-readobj --notes tag.o | FileCheck --check-prefix LLVM-NORMAL %s +# RUN: llvm-readobj --notes tag-short.o | FileCheck --check-prefix LLVM-SHORT %s +# RUN: llvm-readobj --notes tag-long.o | FileCheck --check-prefix LLVM-LONG %s + +# LLVM-SHORT: Notes [ +# LLVM-SHORT-NEXT: NoteSection { +# LLVM-SHORT-NEXT: Name: .note.AARCH64-PAUTH-ABI-tag +# LLVM-SHORT-NEXT: Offset: 0x40 +# LLVM-SHORT-NEXT: Size: 0x1C +# LLVM-SHORT-NEXT: Note { +# LLVM-SHORT-NEXT: Owner: ARM +# LLVM-SHORT-NEXT: Data size: 0xC +# LLVM-SHORT-NEXT: Type: NT_ARM_TYPE_PAUTH_ABI_TAG +# LLVM-SHORT-NEXT: Description data ( +# LLVM-SHORT-NEXT: 0000: 2A000000 00000000 01000000 +# LLVM-SHORT-NEXT: ) +# LLVM-SHORT-NEXT: } +# LLVM-SHORT-NEXT: } +# LLVM-SHORT-NEXT: ] + +# LLVM-NORMAL: Notes [ +# LLVM-NORMAL-NEXT: NoteSection { +# LLVM-NORMAL-NEXT: Name: .note.AARCH64-PAUTH-ABI-tag +# LLVM-NORMAL-NEXT: Offset: 0x40 +# LLVM-NORMAL-NEXT: Size: 0x20 +# LLVM-NORMAL-NEXT: Note { +# LLVM-NORMAL-NEXT: Owner: ARM +# LLVM-NORMAL-NEXT: Data size: 0x10 +# LLVM-NORMAL-NEXT: Type: NT_ARM_TYPE_PAUTH_ABI_TAG +# LLVM-NORMAL-NEXT: Platform: 42 +# LLVM-NORMAL-NEXT: Version: 1 +# LLVM-NORMAL-NEXT: } +# LLVM-NORMAL-NEXT: } +# LLVM-NORMAL-NEXT: ] + +# LLVM-LONG: Notes [ +# LLVM-LONG-NEXT: NoteSection { +# LLVM-LONG-NEXT: Name: .note.AARCH64-PAUTH-ABI-tag +# LLVM-LONG-NEXT: Offset: 0x40 +# LLVM-LONG-NEXT: Size: 0x28 +# LLVM-LONG-NEXT: Note { +# LLVM-LONG-NEXT: Owner: ARM +# LLVM-LONG-NEXT: Data size: 0x18 +# LLVM-LONG-NEXT: Type: NT_ARM_TYPE_PAUTH_ABI_TAG +# LLVM-LONG-NEXT: Platform: 42 +# LLVM-LONG-NEXT: Version: 1 +# LLVM-LONG-NEXT: Additional info: EFCDAB8967452301 +# LLVM-LONG-NEXT: } +# LLVM-LONG-NEXT: } +# LLVM-LONG-NEXT: ] + +#--- abi-tag.s + +.section ".note.AARCH64-PAUTH-ABI-tag", "a" +.long 4 +.long 16 +.long 1 +.asciz "ARM" + +.quad 42 // platform +.quad 1 // version + +#--- abi-tag-short.s + +.section ".note.AARCH64-PAUTH-ABI-tag", "a" +.long 4 +.long 12 +.long 1 +.asciz "ARM" + +.quad 42 +.word 1 + +#--- abi-tag-long.s + +.section ".note.AARCH64-PAUTH-ABI-tag", "a" +.long 4 +.long 24 +.long 1 +.asciz "ARM" + +.quad 42 // platform +.quad 1 // version +.quad 0x0123456789ABCDEF // extra data diff --git a/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test b/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test index 1a1c6dd4d0d1c22415142e6a9b8c31c586463efe..9f18248e7921d7580e9ca33579589f66413f57e1 100644 --- a/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test +++ b/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test @@ -102,7 +102,7 @@ ProgramHeaders: # RUN: llvm-readelf --dyn-relocations %t3 2>&1 | \ # RUN: FileCheck %s -DFILE=%t3 --check-prefix=INVALID-DT-RELAENT --implicit-check-not=warning: -## INVALID-DT-RELAENT: warning: '[[FILE]]': invalid DT_RELASZ value (0x18) or DT_RELAENT value (0xff) +# INVALID-DT-RELAENT: warning: '[[FILE]]': invalid DT_RELASZ value (0x18) or DT_RELAENT value (0xff) ## Show we print a warning for an invalid relocation table size stored in a DT_RELSZ entry. @@ -112,7 +112,7 @@ ProgramHeaders: # RUN: llvm-readobj --dyn-relocations %t4a 2>&1 | FileCheck %s -DFILE=%t4a --check-prefix=INVALID-DT-RELSZ1 # RUN: llvm-readelf --dyn-relocations %t4a 2>&1 | FileCheck %s -DFILE=%t4a --check-prefix=INVALID-DT-RELSZ1 -## INVALID-DT-RELSZ1: warning: '[[FILE]]': invalid DT_RELSZ value (0xff) or DT_RELENT value (0x18) +# INVALID-DT-RELSZ1: warning: '[[FILE]]': invalid DT_RELSZ value (0xff) or DT_RELENT value (0x18) ## Case B: the DT_RELSZ has value of 0x251, what is too large, because the relocation table goes past the EOF. @@ -127,7 +127,7 @@ ProgramHeaders: # RUN: llvm-readobj --dyn-relocations %t5 2>&1 | FileCheck %s -DFILE=%t5 --check-prefix=INVALID-DT-RELENT # RUN: llvm-readelf --dyn-relocations %t5 2>&1 | FileCheck %s -DFILE=%t5 --check-prefix=INVALID-DT-RELENT -## INVALID-DT-RELENT: warning: '[[FILE]]': invalid DT_RELSZ value (0x18) or DT_RELENT value (0xff) +# INVALID-DT-RELENT: warning: '[[FILE]]': invalid DT_RELSZ value (0x18) or DT_RELENT value (0xff) ## Show we print a warning for an invalid relocation table size stored in a DT_RELRSZ/DT_ANDROID_RELRSZ entry. # RUN: yaml2obj --docnum=2 -DRELTYPE=RELR -DTAG1=DT_RELRSZ -DTAG1VAL=0xFF -DTAG2=DT_RELRENT %s -o %t6 @@ -138,8 +138,8 @@ ProgramHeaders: # RUN: llvm-readobj --dyn-relocations %t7 2>&1 | FileCheck %s -DFILE=%t7 --check-prefix=INVALID-DT-ANDROID-RELRSZ # RUN: llvm-readelf --dyn-relocations %t7 2>&1 | FileCheck %s -DFILE=%t7 --check-prefix=INVALID-DT-ANDROID-RELRSZ -## INVALID-DT-RELRSZ: warning: '[[FILE]]': invalid DT_RELRSZ value (0xff) or DT_RELRENT value (0x18) -## INVALID-DT-ANDROID-RELRSZ: warning: '[[FILE]]': invalid DT_ANDROID_RELRSZ value (0xff) or DT_ANDROID_RELRENT value (0x18) +# INVALID-DT-RELRSZ: warning: '[[FILE]]': invalid DT_RELRSZ value (0xff) or DT_RELRENT value (0x18) +# INVALID-DT-ANDROID-RELRSZ: warning: '[[FILE]]': invalid DT_ANDROID_RELRSZ value (0xff) or DT_ANDROID_RELRENT value (0x18) ## Show we print a warning for an invalid relocation table entry size stored in a DT_RELRENT/DT_ANDROID_RELRENT entry. # RUN: yaml2obj --docnum=2 -DRELTYPE=RELR -DTAG1=DT_RELRSZ -DTAG2=DT_RELRENT -DTAG2VAL=0xFF %s -o %t8 @@ -149,8 +149,8 @@ ProgramHeaders: # RUN: llvm-readobj --dyn-relocations %t9 2>&1 | FileCheck %s -DFILE=%t9 --check-prefix=INVALID-DT-ANDROID-RELRENT # RUN: llvm-readelf --dyn-relocations %t9 2>&1 | FileCheck %s -DFILE=%t9 --check-prefix=INVALID-DT-ANDROID-RELRENT -## INVALID-DT-RELRENT: invalid DT_RELRSZ value (0x18) or DT_RELRENT value (0xff) -## INVALID-DT-ANDROID-RELRENT: invalid DT_ANDROID_RELRSZ value (0x18) or DT_ANDROID_RELRENT value (0xff) +# INVALID-DT-RELRENT: invalid DT_RELRSZ value (0x18) or DT_RELRENT value (0xff) +# INVALID-DT-ANDROID-RELRENT: invalid DT_ANDROID_RELRSZ value (0x18) or DT_ANDROID_RELRENT value (0xff) ## Show we print a warning for an invalid value of DT_PLTRELSZ, which describes the total size ## of the relocation entries associated with the procedure linkage table. @@ -459,3 +459,43 @@ ProgramHeaders: - Type: PT_DYNAMIC FirstSec: .dynamic LastSec: .dynamic + +## Show we print a warning for an invalid relocation table size stored in a DT_AARCH64_AUTH_RELRSZ entry. +# RUN: yaml2obj --docnum=8 -DRELTYPE=RELR -DTAG1=DT_AARCH64_AUTH_RELRSZ -DTAG1VAL=0xFF -DTAG2=DT_AARCH64_AUTH_RELRENT %s -o %t14 +# RUN: llvm-readobj --dyn-relocations %t14 2>&1 | FileCheck %s -DFILE=%t14 --check-prefix=INVALID-DT-AARCH64-AUTH-RELRSZ +# RUN: llvm-readelf --dyn-relocations %t14 2>&1 | FileCheck %s -DFILE=%t14 --check-prefix=INVALID-DT-AARCH64-AUTH-RELRSZ + +# INVALID-DT-AARCH64-AUTH-RELRSZ: warning: '[[FILE]]': invalid DT_AARCH64_AUTH_RELRSZ value (0xff) or DT_AARCH64_AUTH_RELRENT value (0x18) + +## Show we print a warning for an invalid relocation table entry size stored in a DT_AARCH64_AUTH_RELRENT entry. +# RUN: yaml2obj --docnum=8 -DRELTYPE=RELR -DTAG1=DT_AARCH64_AUTH_RELRSZ -DTAG2=DT_AARCH64_AUTH_RELRENT -DTAG2VAL=0xFF %s -o %t15 +# RUN: llvm-readobj --dyn-relocations %t15 2>&1 | FileCheck %s -DFILE=%t15 --check-prefix=INVALID-DT-AARCH64-AUTH-RELRENT +# RUN: llvm-readelf --dyn-relocations %t15 2>&1 | FileCheck %s -DFILE=%t15 --check-prefix=INVALID-DT-AARCH64-AUTH-RELRENT + +# INVALID-DT-AARCH64-AUTH-RELRENT: invalid DT_AARCH64_AUTH_RELRSZ value (0x18) or DT_AARCH64_AUTH_RELRENT value (0xff) + +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_DYN + Machine: EM_AARCH64 +Sections: + - Name: .relx.dyn + Type: SHT_[[RELTYPE]] + - Name: .dynamic + Type: SHT_DYNAMIC + Entries: + - Tag: DT_[[RELTYPE]] + Value: 0x0 + - Tag: [[TAG1]] + Value: [[TAG1VAL=0x18]] + - Tag: [[TAG2]] + Value: [[TAG2VAL=0x18]] + - Tag: DT_NULL + Value: 0x0 +DynamicSymbols: [] +ProgramHeaders: + - Type: PT_LOAD + FirstSec: .relx.dyn + LastSec: .dynamic diff --git a/llvm/test/tools/llvm-readobj/ELF/dynamic-tags-machine-specific.test b/llvm/test/tools/llvm-readobj/ELF/dynamic-tags-machine-specific.test index c32ea33b9b3cbf62a94664c4149624aedbea143b..e7bd9cf9b48b233deb1d90b9f0c707451236982f 100644 --- a/llvm/test/tools/llvm-readobj/ELF/dynamic-tags-machine-specific.test +++ b/llvm/test/tools/llvm-readobj/ELF/dynamic-tags-machine-specific.test @@ -355,20 +355,26 @@ ProgramHeaders: # RUN: llvm-readobj --dynamic-table %t.aarch64 | FileCheck %s --check-prefix=LLVM-AARCH64 # RUN: llvm-readelf --dynamic-table %t.aarch64 | FileCheck %s --check-prefix=GNU-AARCH64 -# LLVM-AARCH64: DynamicSection [ (4 entries) +# LLVM-AARCH64: DynamicSection [ (7 entries) # LLVM-AARCH64-NEXT: Tag Type Name/Value -# LLVM-AARCH64-NEXT: 0x0000000070000001 AARCH64_BTI_PLT 1 -# LLVM-AARCH64-NEXT: 0x0000000070000003 AARCH64_PAC_PLT 2 -# LLVM-AARCH64-NEXT: 0x0000000070000005 AARCH64_VARIANT_PCS 3 -# LLVM-AARCH64-NEXT: 0x0000000000000000 NULL 0x0 +# LLVM-AARCH64-NEXT: 0x0000000070000001 AARCH64_BTI_PLT 1 +# LLVM-AARCH64-NEXT: 0x0000000070000003 AARCH64_PAC_PLT 2 +# LLVM-AARCH64-NEXT: 0x0000000070000005 AARCH64_VARIANT_PCS 3 +# LLVM-AARCH64-NEXT: 0x0000000070000012 AARCH64_AUTH_RELR 0x4 +# LLVM-AARCH64-NEXT: 0x0000000070000011 AARCH64_AUTH_RELRSZ 5 +# LLVM-AARCH64-NEXT: 0x0000000070000013 AARCH64_AUTH_RELRENT 6 +# LLVM-AARCH64-NEXT: 0x0000000000000000 NULL 0x0 # LLVM-AARCH64-NEXT:] -# GNU-AARCH64: Dynamic section at offset {{.*}} contains 4 entries: +# GNU-AARCH64: Dynamic section at offset {{.*}} contains 7 entries: # GNU-AARCH64-NEXT: Tag Type Name/Value -# GNU-AARCH64-NEXT: 0x0000000070000001 (AARCH64_BTI_PLT) 1 -# GNU-AARCH64-NEXT: 0x0000000070000003 (AARCH64_PAC_PLT) 2 -# GNU-AARCH64-NEXT: 0x0000000070000005 (AARCH64_VARIANT_PCS) 3 -# GNU-AARCH64-NEXT: 0x0000000000000000 (NULL) 0x0 +# GNU-AARCH64-NEXT: 0x0000000070000001 (AARCH64_BTI_PLT) 1 +# GNU-AARCH64-NEXT: 0x0000000070000003 (AARCH64_PAC_PLT) 2 +# GNU-AARCH64-NEXT: 0x0000000070000005 (AARCH64_VARIANT_PCS) 3 +# GNU-AARCH64-NEXT: 0x0000000070000012 (AARCH64_AUTH_RELR) 0x4 +# GNU-AARCH64-NEXT: 0x0000000070000011 (AARCH64_AUTH_RELRSZ) 5 +# GNU-AARCH64-NEXT: 0x0000000070000013 (AARCH64_AUTH_RELRENT) 6 +# GNU-AARCH64-NEXT: 0x0000000000000000 (NULL) 0x0 --- !ELF FileHeader: @@ -386,6 +392,12 @@ Sections: Value: 2 - Tag: DT_AARCH64_VARIANT_PCS Value: 3 + - Tag: DT_AARCH64_AUTH_RELR + Value: 4 + - Tag: DT_AARCH64_AUTH_RELRSZ + Value: 5 + - Tag: DT_AARCH64_AUTH_RELRENT + Value: 6 - Tag: DT_NULL Value: 0 ProgramHeaders: diff --git a/llvm/test/tools/llvm-readobj/ELF/machine-specific-section-types.test b/llvm/test/tools/llvm-readobj/ELF/machine-specific-section-types.test index 99fafe35fa4433db5bb41a9ea7f3f8aa039cc98a..f9524383e80b6dbec626ee451aca6a49c9d7f472 100644 --- a/llvm/test/tools/llvm-readobj/ELF/machine-specific-section-types.test +++ b/llvm/test/tools/llvm-readobj/ELF/machine-specific-section-types.test @@ -53,11 +53,14 @@ # MIPS-GNU: abiflags MIPS_ABIFLAGS # MIPS-GNU: dwarf MIPS_DWARF +# AARCH64-LLVM: Name: aarch64_auth_relr +# AARCH64-LLVM: Type: SHT_AARCH64_AUTH_RELR # AARCH64-LLVM: Name: .memtag.globals.dynamic # AARCH64-LLVM: Type: SHT_AARCH64_MEMTAG_GLOBALS_DYNAMIC # AARCH64-LLVM: Name: .memtag.globals.static # AARCH64-LLVM: Type: SHT_AARCH64_MEMTAG_GLOBALS_STATIC +# AARCH64-GNU: aarch64_auth_relr AARCH64_AUTH_RELR # AARCH64-GNU: .memtag.globals.dynamic AARCH64_MEMTAG_GLOBALS_DYNAMIC # AARCH64-GNU: .memtag.globals.static AARCH64_MEMTAG_GLOBALS_STATIC @@ -113,6 +116,8 @@ FileHeader: Type: ET_REL Machine: EM_AARCH64 Sections: + - Name: aarch64_auth_relr + Type: SHT_AARCH64_AUTH_RELR - Name: .memtag.globals.dynamic Type: SHT_AARCH64_MEMTAG_GLOBALS_DYNAMIC - Name: .memtag.globals.static diff --git a/llvm/test/tools/llvm-readobj/ELF/needed-libs.test b/llvm/test/tools/llvm-readobj/ELF/needed-libs.test index d37ab6238e137d6552a2a1fd784c61212ae60b0a..d2d0bdad26ed05e33f8bbcbb23603496c22ccecc 100644 --- a/llvm/test/tools/llvm-readobj/ELF/needed-libs.test +++ b/llvm/test/tools/llvm-readobj/ELF/needed-libs.test @@ -5,6 +5,8 @@ # RUN: | FileCheck %s -DFILE=%t1 --implicit-check-not=warning: --strict-whitespace --check-prefix=NEEDED-LIBS # RUN: llvm-readelf --needed-libs %t1 2>&1 \ # RUN: | FileCheck %s -DFILE=%t1 --implicit-check-not=warning: --strict-whitespace --check-prefix=NEEDED-LIBS +# RUN: llvm-readobj --elf-output-style=JSON --needed-libs --pretty-print %t1 2>&1 \ +# RUN: | FileCheck %s -DFILE=%t1 --implicit-check-not=warning: --match-full-lines --strict-whitespace --check-prefix=JSON ## Check that library names are sorted when printed. ## Document that we also sort error entries. @@ -19,6 +21,17 @@ # NEEDED-LIBS-NEXT:{{^}} ccc{{$}} # NEEDED-LIBS-NEXT:{{^}}]{{$}} +# JSON: "FileSummary": { +# JSON: "NeededLibraries": [{{.*}}warning: '[[FILE]]': string table at offset 0x78: unable to read the string at 0x9999a11: it goes past the end of the table (0x85) +# JSON-NEXT:{{.*}}warning: '[[FILE]]': string table at offset 0x78: unable to read the string at 0x1111189: it goes past the end of the table (0x85) +# JSON-EMPTY: +# JSON-NEXT: "", +# JSON-NEXT: "", +# JSON-NEXT: "aaa", +# JSON-NEXT: "bbb", +# JSON-NEXT: "ccc" +# JSON-NEXT: ] + --- !ELF FileHeader: Class: ELFCLASS64 diff --git a/llvm/test/tools/llvm-readobj/ELF/relr-relocs.test b/llvm/test/tools/llvm-readobj/ELF/relr-relocs.test index 3bb54b1adc1f4fe148145a6a2c3979772a1c2843..91b148ebb6e3c83786d07748d085fde9efe51df7 100644 --- a/llvm/test/tools/llvm-readobj/ELF/relr-relocs.test +++ b/llvm/test/tools/llvm-readobj/ELF/relr-relocs.test @@ -158,7 +158,7 @@ Sections: Link: [[LINK=]] ## Check we report a warning when we are unable to dump relocations -## for a SHT_RELR/SHT_ANDROID_RELR section. +## for a SHT_RELR/SHT_ANDROID_RELR/SHT_AARCH64_AUTH_RELR section. ## Case A: check the case when relocations can't be read from an SHT_RELR section. # RUN: yaml2obj --docnum=2 -DENTSIZE=1 %s -o %t2.broken @@ -186,7 +186,20 @@ Sections: # RUN: llvm-readelf --relocations %t2.broken.android 2>&1 | \ # RUN: FileCheck -DFILE=%t2.broken.android --check-prefix=BROKEN-GNU %s -DSECNAME=SHT_ANDROID_RELR -## Check the behavior when the sh_link field of the SHT_RELR/SHT_ANDROID_RELR section +## Case C: check the case when we have an SHT_AARCH64_AUTH_RELR section in non-AArch64 ELF. +## SHT_AARCH64_AUTH_RELR = 0x70000004. +# RUN: yaml2obj --docnum=2 -DENTSIZE=1 -DSHTYPE=0x70000004 %s -o %t2.broken.aarch64auth +# RUN: llvm-readobj --relocations %t2.broken.aarch64auth 2>&1 | \ +# RUN: FileCheck -DFILE=%t2.broken.aarch64auth --check-prefix=WRONGARCH-LLVM-AARCH64-AUTH %s -DSECNAME=SHT_AARCH64_AUTH_RELR +# RUN: llvm-readelf --relocations %t2.broken.aarch64auth 2>&1 | \ +# RUN: FileCheck -DFILE=%t2.broken.aarch64auth --check-prefix=WRONGARCH-GNU-AARCH64-AUTH %s -DSECNAME=SHT_AARCH64_AUTH_RELR + +# WRONGARCH-LLVM-AARCH64-AUTH: Relocations [ +# WRONGARCH-LLVM-AARCH64-AUTH-NEXT: ] + +# WRONGARCH-GNU-AARCH64-AUTH-NOT: Relocation section + +## Check the behavior when the sh_link field of the SHT_RELR/SHT_ANDROID_RELR/SHT_AARCH64_AUTH_RELR section ## is set to an arbitrary value. Normally, it is set to 0, because such sections contains ## only relative relocations and do not have an associated symbol table, like other ## relocation sections. diff --git a/llvm/test/tools/obj2yaml/XCOFF/aix.yaml b/llvm/test/tools/obj2yaml/XCOFF/aix.yaml index cd1e88dec11d29b97a8f0482ac07673374b51faa..fbd5fa0629d10bd33dfe59a7df5bd2fb8065625d 100644 --- a/llvm/test/tools/obj2yaml/XCOFF/aix.yaml +++ b/llvm/test/tools/obj2yaml/XCOFF/aix.yaml @@ -52,12 +52,30 @@ # CHECK32-NEXT: Type: 0x0 # CHECK32-NEXT: StorageClass: C_EXT # CHECK32-NEXT: NumberOfAuxEntries: 1 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_CSECT +# CHECK32-NEXT: ParameterHashIndex: 0 +# CHECK32-NEXT: TypeChkSectNum: 0 +# CHECK32-NEXT: SymbolAlignmentAndType: 0 +# CHECK32-NEXT: StorageMappingClass: XMC_PR +# CHECK32-NEXT: SectionOrLength: 0 +# CHECK32-NEXT: StabInfoIndex: 0 +# CHECK32-NEXT: StabSectNum: 0 # CHECK32-NEXT: - Name: .data # CHECK32-NEXT: Value: 0x70 # CHECK32-NEXT: Section: .data # CHECK32-NEXT: Type: 0x0 # CHECK32-NEXT: StorageClass: C_HIDEXT # CHECK32-NEXT: NumberOfAuxEntries: 1 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_CSECT +# CHECK32-NEXT: ParameterHashIndex: 0 +# CHECK32-NEXT: TypeChkSectNum: 0 +# CHECK32-NEXT: SymbolAlignmentAndType: 0 +# CHECK32-NEXT: StorageMappingClass: XMC_PR +# CHECK32-NEXT: SectionOrLength: 0 +# CHECK32-NEXT: StabInfoIndex: 0 +# CHECK32-NEXT: StabSectNum: 0 # CHECK64: --- !XCOFF # CHECK64-NEXT: FileHeader: @@ -106,12 +124,28 @@ # CHECK64-NEXT: Type: 0x0 # CHECK64-NEXT: StorageClass: C_EXT # CHECK64-NEXT: NumberOfAuxEntries: 1 +# CHECK64-NEXT: AuxEntries: +# CHECK64-NEXT: - Type: AUX_CSECT +# CHECK64-NEXT: ParameterHashIndex: 0 +# CHECK64-NEXT: TypeChkSectNum: 0 +# CHECK64-NEXT: SymbolAlignmentAndType: 0 +# CHECK64-NEXT: StorageMappingClass: XMC_PR +# CHECK64-NEXT: SectionOrLengthLo: 0 +# CHECK64-NEXT: SectionOrLengthHi: 0 # CHECK64-NEXT: - Name: .data # CHECK64-NEXT: Value: 0x70 # CHECK64-NEXT: Section: .data # CHECK64-NEXT: Type: 0x0 # CHECK64-NEXT: StorageClass: C_HIDEXT # CHECK64-NEXT: NumberOfAuxEntries: 1 +# CHECK64-NEXT: AuxEntries: +# CHECK64-NEXT: - Type: AUX_CSECT +# CHECK64-NEXT: ParameterHashIndex: 0 +# CHECK64-NEXT: TypeChkSectNum: 0 +# CHECK64-NEXT: SymbolAlignmentAndType: 0 +# CHECK64-NEXT: StorageMappingClass: XMC_PR +# CHECK64-NEXT: SectionOrLengthLo: 0 +# CHECK64-NEXT: SectionOrLengthHi: 0 --- !XCOFF FileHeader: @@ -140,9 +174,13 @@ Symbols: Type: 0x0 StorageClass: C_EXT NumberOfAuxEntries: 1 + AuxEntries: + - Type: AUX_CSECT - Name: .data Value: 0x70 Section: .data Type: 0x0 StorageClass: C_HIDEXT NumberOfAuxEntries: 1 + AuxEntries: + - Type: AUX_CSECT diff --git a/llvm/test/tools/obj2yaml/XCOFF/aux-symbols.yaml b/llvm/test/tools/obj2yaml/XCOFF/aux-symbols.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f93b8dae0ca9b50e64065597162b2c7731690f2 --- /dev/null +++ b/llvm/test/tools/obj2yaml/XCOFF/aux-symbols.yaml @@ -0,0 +1,322 @@ +## Check that obj2yaml can parse auxiliary symbols for XCOFF object file correctly. + +## 32-bit +# RUN: yaml2obj %s --docnum=1 -o %t32 +# RUN: obj2yaml %t32 | FileCheck %s --check-prefix=CHECK32 + +# CHECK32: --- !XCOFF +# CHECK32-NEXT: FileHeader: +# CHECK32-NEXT: MagicNumber: 0x1DF +# CHECK32-NEXT: NumberOfSections: 0 +# CHECK32-NEXT: CreationTime: 0 +# CHECK32-NEXT: OffsetToSymbolTable: 0x14 +# CHECK32-NEXT: EntriesInSymbolTable: 13 +# CHECK32-NEXT: AuxiliaryHeaderSize: 0 +# CHECK32-NEXT: Flags: 0x0 +# CHECK32-NEXT: Symbols: +# CHECK32-NEXT: - Name: '' +# CHECK32-NEXT: Value: 0x0 +# CHECK32-NEXT: Section: N_UNDEF +# CHECK32-NEXT: Type: 0x0 +# CHECK32-NEXT: StorageClass: C_FILE +# CHECK32-NEXT: NumberOfAuxEntries: 1 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_FILE +# CHECK32-NEXT: FileNameOrString: FileName +# CHECK32-NEXT: FileStringType: XFT_CD +# CHECK32-NEXT: - Name: '' +# CHECK32-NEXT: Value: 0x0 +# CHECK32-NEXT: Section: N_UNDEF +# CHECK32-NEXT: Type: 0x0 +# CHECK32-NEXT: StorageClass: C_HIDEXT +# CHECK32-NEXT: NumberOfAuxEntries: 1 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_CSECT +# CHECK32-NEXT: ParameterHashIndex: 1 +# CHECK32-NEXT: TypeChkSectNum: 2 +# CHECK32-NEXT: SymbolAlignmentAndType: 41 +# CHECK32-NEXT: StorageMappingClass: XMC_PR +# CHECK32-NEXT: SectionOrLength: 3 +# CHECK32-NEXT: StabInfoIndex: 4 +# CHECK32-NEXT: StabSectNum: 5 +# CHECK32-NEXT: - Name: '' +# CHECK32-NEXT: Value: 0x0 +# CHECK32-NEXT: Section: N_UNDEF +# CHECK32-NEXT: Type: 0x0 +# CHECK32-NEXT: StorageClass: C_EXT +# CHECK32-NEXT: NumberOfAuxEntries: 2 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_FCN +# CHECK32-NEXT: OffsetToExceptionTbl: 1 +# CHECK32-NEXT: SizeOfFunction: 2 +# CHECK32-NEXT: SymIdxOfNextBeyond: 3 +# CHECK32-NEXT: PtrToLineNum: 4 +# CHECK32-NEXT: - Type: AUX_CSECT +# CHECK32-NEXT: ParameterHashIndex: 1 +# CHECK32-NEXT: TypeChkSectNum: 2 +# CHECK32-NEXT: SymbolAlignmentAndType: 17 +# CHECK32-NEXT: StorageMappingClass: XMC_PR +# CHECK32-NEXT: SectionOrLength: 4 +# CHECK32-NEXT: StabInfoIndex: 5 +# CHECK32-NEXT: StabSectNum: 6 +# CHECK32-NEXT: - Name: '' +# CHECK32-NEXT: Value: 0x0 +# CHECK32-NEXT: Section: N_UNDEF +# CHECK32-NEXT: Type: 0x0 +# CHECK32-NEXT: StorageClass: C_DWARF +# CHECK32-NEXT: NumberOfAuxEntries: 1 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_SECT +# CHECK32-NEXT: LengthOfSectionPortion: 44 +# CHECK32-NEXT: NumberOfRelocEnt: 1 +# CHECK32-NEXT: - Name: '' +# CHECK32-NEXT: Value: 0x0 +# CHECK32-NEXT: Section: N_UNDEF +# CHECK32-NEXT: Type: 0x0 +# CHECK32-NEXT: StorageClass: C_STAT +# CHECK32-NEXT: NumberOfAuxEntries: 1 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_STAT +# CHECK32-NEXT: SectionLength: 1 +# CHECK32-NEXT: NumberOfRelocEnt: 2 +# CHECK32-NEXT: NumberOfLineNum: 3 +# CHECK32-NEXT: - Name: '' +# CHECK32-NEXT: Value: 0x0 +# CHECK32-NEXT: Section: N_UNDEF +# CHECK32-NEXT: Type: 0x0 +# CHECK32-NEXT: StorageClass: C_BLOCK +# CHECK32-NEXT: NumberOfAuxEntries: 1 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_SYM +# CHECK32-NEXT: LineNumHi: 1 +# CHECK32-NEXT: LineNumLo: 2 + +--- !XCOFF +FileHeader: + MagicNumber: 0x01DF +Symbols: + - StorageClass: C_FILE + AuxEntries: + - Type: AUX_FILE + FileNameOrString: FileName + FileStringType: XFT_CD + - StorageClass: C_HIDEXT + AuxEntries: + - Type: AUX_CSECT + ParameterHashIndex: 1 + TypeChkSectNum: 2 + SymbolAlignmentAndType: 41 + SectionOrLength: 3 + StabInfoIndex: 4 + StabSectNum: 5 + - StorageClass: C_EXT + AuxEntries: + - Type: AUX_FCN + OffsetToExceptionTbl: 1 + SizeOfFunction: 2 + SymIdxOfNextBeyond: 3 + PtrToLineNum: 4 + - Type: AUX_CSECT + ParameterHashIndex: 1 + TypeChkSectNum: 2 + SymbolAlignmentAndType: 17 + SectionOrLength: 4 + StabInfoIndex: 5 + StabSectNum: 6 + - StorageClass: C_DWARF + AuxEntries: + - Type: AUX_SECT + LengthOfSectionPortion: 44 + NumberOfRelocEnt: 1 + - StorageClass: C_STAT + AuxEntries: + - Type: AUX_STAT + SectionLength: 1 + NumberOfRelocEnt: 2 + NumberOfLineNum: 3 + - StorageClass: C_BLOCK + AuxEntries: + - Type: AUX_SYM + LineNumHi: 1 + LineNumLo: 2 + +## 64-bit +# RUN: yaml2obj %s --docnum=2 -o %t64 +# RUN: obj2yaml %t64 | FileCheck %s --check-prefix=CHECK64 + +# CHECK64: --- !XCOFF +# CHECK64-NEXT: FileHeader: +# CHECK64-NEXT: MagicNumber: 0x1F7 +# CHECK64-NEXT: NumberOfSections: 0 +# CHECK64-NEXT: CreationTime: 0 +# CHECK64-NEXT: OffsetToSymbolTable: 0x18 +# CHECK64-NEXT: EntriesInSymbolTable: 12 +# CHECK64-NEXT: AuxiliaryHeaderSize: 0 +# CHECK64-NEXT: Flags: 0x0 +# CHECK64-NEXT: Symbols: +# CHECK64-NEXT: - Name: '' +# CHECK64-NEXT: Value: 0x0 +# CHECK64-NEXT: Section: N_UNDEF +# CHECK64-NEXT: Type: 0x0 +# CHECK64-NEXT: StorageClass: C_FILE +# CHECK64-NEXT: NumberOfAuxEntries: 1 +# CHECK64-NEXT: AuxEntries: +# CHECK64-NEXT: - Type: AUX_FILE +# CHECK64-NEXT: FileNameOrString: FileName +# CHECK64-NEXT: FileStringType: XFT_CD +# CHECK64-NEXT: - Name: '' +# CHECK64-NEXT: Value: 0x0 +# CHECK64-NEXT: Section: N_UNDEF +# CHECK64-NEXT: Type: 0x0 +# CHECK64-NEXT: StorageClass: C_HIDEXT +# CHECK64-NEXT: NumberOfAuxEntries: 1 +# CHECK64-NEXT: AuxEntries: +# CHECK64-NEXT: - Type: AUX_CSECT +# CHECK64-NEXT: ParameterHashIndex: 1 +# CHECK64-NEXT: TypeChkSectNum: 2 +# CHECK64-NEXT: SymbolAlignmentAndType: 41 +# CHECK64-NEXT: StorageMappingClass: XMC_PR +# CHECK64-NEXT: SectionOrLengthLo: 3 +# CHECK64-NEXT: SectionOrLengthHi: 4 +# CHECK64-NEXT: - Name: '' +# CHECK64-NEXT: Value: 0x0 +# CHECK64-NEXT: Section: N_UNDEF +# CHECK64-NEXT: Type: 0x0 +# CHECK64-NEXT: StorageClass: C_EXT +# CHECK64-NEXT: NumberOfAuxEntries: 3 +# CHECK64-NEXT: AuxEntries: +# CHECK64-NEXT: - Type: AUX_FCN +# CHECK64-NEXT: SizeOfFunction: 3 +# CHECK64-NEXT: SymIdxOfNextBeyond: 2 +# CHECK64-NEXT: PtrToLineNum: 1 +# CHECK64-NEXT: - Type: AUX_EXCEPT +# CHECK64-NEXT: OffsetToExceptionTbl: 1 +# CHECK64-NEXT: SizeOfFunction: 2 +# CHECK64-NEXT: SymIdxOfNextBeyond: 3 +# CHECK64-NEXT: - Type: AUX_CSECT +# CHECK64-NEXT: ParameterHashIndex: 1 +# CHECK64-NEXT: TypeChkSectNum: 2 +# CHECK64-NEXT: SymbolAlignmentAndType: 17 +# CHECK64-NEXT: StorageMappingClass: XMC_PR +# CHECK64-NEXT: SectionOrLengthLo: 3 +# CHECK64-NEXT: SectionOrLengthHi: 4 +# CHECK64-NEXT: - Name: '' +# CHECK64-NEXT: Value: 0x0 +# CHECK64-NEXT: Section: N_UNDEF +# CHECK64-NEXT: Type: 0x0 +# CHECK64-NEXT: StorageClass: C_DWARF +# CHECK64-NEXT: NumberOfAuxEntries: 1 +# CHECK64-NEXT: AuxEntries: +# CHECK64-NEXT: - Type: AUX_SECT +# CHECK64-NEXT: LengthOfSectionPortion: 44 +# CHECK64-NEXT: NumberOfRelocEnt: 1 +# CHECK64-NEXT: - Name: '' +# CHECK64-NEXT: Value: 0x0 +# CHECK64-NEXT: Section: N_UNDEF +# CHECK64-NEXT: Type: 0x0 +# CHECK64-NEXT: StorageClass: C_BLOCK +# CHECK64-NEXT: NumberOfAuxEntries: 1 +# CHECK64-NEXT: AuxEntries: +# CHECK64-NEXT: - Type: AUX_SYM +# CHECK64-NEXT: LineNum: 1 + +--- !XCOFF +FileHeader: + MagicNumber: 0x1F7 +Symbols: + - StorageClass: C_FILE + AuxEntries: + - Type: AUX_FILE + FileNameOrString: FileName + FileStringType: XFT_CD + - StorageClass: C_HIDEXT + AuxEntries: + - Type: AUX_CSECT + ParameterHashIndex: 1 + TypeChkSectNum: 2 + SymbolAlignmentAndType: 41 + SectionOrLengthLo: 3 + SectionOrLengthHi: 4 + - StorageClass: C_EXT + AuxEntries: + - Type: AUX_FCN + SizeOfFunction: 3 + SymIdxOfNextBeyond: 2 + PtrToLineNum: 1 + - Type: AUX_EXCEPT + OffsetToExceptionTbl: 1 + SizeOfFunction: 2 + SymIdxOfNextBeyond: 3 + - Type: AUX_CSECT + ParameterHashIndex: 1 + TypeChkSectNum: 2 + SymbolAlignmentAndType: 17 + SectionOrLengthLo: 3 + SectionOrLengthHi: 4 + - StorageClass: C_DWARF + AuxEntries: + - Type: AUX_SECT + LengthOfSectionPortion: 44 + NumberOfRelocEnt: 1 + - StorageClass: C_BLOCK + AuxEntries: + - Type: AUX_SYM + LineNum: 1 + +## Check that error messages are reported appropriately. + +# RUN: yaml2obj %s --docnum=3 -o - | not obj2yaml 2>&1 | FileCheck %s --check-prefix=ERROR1 +# ERROR1: failed to parse symbol "SYM_STAT" with index of 1: expected 1 aux symbol for C_STAT, while got 2 + +--- !XCOFF +FileHeader: + MagicNumber: 0x1DF +Symbols: + - Name: SYM + - Name: SYM_STAT + StorageClass: C_STAT + NumberOfAuxEntries: 2 + AuxEntries: + - Type: AUX_STAT + - Type: AUX_STAT + +# RUN: yaml2obj %s --docnum=4 -o - | not obj2yaml 2>&1 | FileCheck %s --check-prefix=ERROR2 +# ERROR2: failed to parse symbol "SYM_BLOCK" with index of 0: expected 1 aux symbol for C_BLOCK or C_FCN, while got 2 + +--- !XCOFF +FileHeader: + MagicNumber: 0x1DF +Symbols: + - Name: SYM_BLOCK + StorageClass: C_BLOCK + NumberOfAuxEntries: 2 + AuxEntries: + - Type: AUX_SYM + - Type: AUX_SYM + +# RUN: yaml2obj %s --docnum=5 -o - | not obj2yaml 2>&1 | FileCheck %s --check-prefix=ERROR3 +# ERROR3: failed to parse symbol "SYM_DWARF" with index of 0: expected 1 aux symbol for C_DWARF, while got 2 + +--- !XCOFF +FileHeader: + MagicNumber: 0x1DF +Symbols: + - Name: SYM_DWARF + StorageClass: C_DWARF + NumberOfAuxEntries: 2 + AuxEntries: + - Type: AUX_SECT + - Type: AUX_SECT + +# RUN: yaml2obj %s --docnum=6 -o - | not obj2yaml 2>&1 | FileCheck %s --check-prefix=ERROR4 +# ERROR4: failed to parse symbol "SYM_EXT" with index of 0: invalid auxiliary symbol type: 252 + +--- !XCOFF +FileHeader: + MagicNumber: 0x1F7 +Symbols: + - Name: SYM_EXT + StorageClass: C_EXT + AuxEntries: + - Type: AUX_FILE + - Type: AUX_CSECT diff --git a/llvm/tools/bugpoint-passes/CMakeLists.txt b/llvm/tools/bugpoint-passes/CMakeLists.txt index 6df49d7abd88098ac779e9609669eda1b0754c61..60fc1bde513710becd7ff1822d7783ec8afdd9ee 100644 --- a/llvm/tools/bugpoint-passes/CMakeLists.txt +++ b/llvm/tools/bugpoint-passes/CMakeLists.txt @@ -10,7 +10,7 @@ if( NOT LLVM_REQUIRES_RTTI ) endif() endif() -if(WIN32 OR CYGWIN) +if(WIN32 OR CYGWIN OR ZOS) set(LLVM_LINK_COMPONENTS Core Support) endif() diff --git a/llvm/tools/dsymutil/DebugMap.cpp b/llvm/tools/dsymutil/DebugMap.cpp index dcdecdfe8210b114f56f20a21111c751f7dc886d..8724b70422f326071ebe30e842f95033fa08713c 100644 --- a/llvm/tools/dsymutil/DebugMap.cpp +++ b/llvm/tools/dsymutil/DebugMap.cpp @@ -287,7 +287,7 @@ MappingTraits::YamlDMO::denormalize(IO &IO) { } uint8_t Type = MachO::N_OSO; - if (Path.endswith(".dylib")) { + if (Path.ends_with(".dylib")) { // FIXME: find a more resilient way Type = MachO::N_LIB; } diff --git a/llvm/tools/llvm-c-test/echo.cpp b/llvm/tools/llvm-c-test/echo.cpp index e2617583ff9bab24da33669139bc3e22ec235679..bfc14e85a12bff960003cd22b15c49db0dc18e3c 100644 --- a/llvm/tools/llvm-c-test/echo.cpp +++ b/llvm/tools/llvm-c-test/echo.cpp @@ -548,16 +548,26 @@ struct FunCloner { break; case LLVMInvoke: { SmallVector Args; - int ArgCount = LLVMGetNumArgOperands(Src); - for (int i = 0; i < ArgCount; i++) + SmallVector Bundles; + unsigned ArgCount = LLVMGetNumArgOperands(Src); + for (unsigned i = 0; i < ArgCount; ++i) Args.push_back(CloneValue(LLVMGetOperand(Src, i))); + unsigned BundleCount = LLVMGetNumOperandBundles(Src); + for (unsigned i = 0; i < BundleCount; ++i) { + auto Bundle = LLVMGetOperandBundleAtIndex(Src, i); + Bundles.push_back(CloneOB(Bundle)); + LLVMDisposeOperandBundle(Bundle); + } LLVMTypeRef FnTy = CloneType(LLVMGetCalledFunctionType(Src)); LLVMValueRef Fn = CloneValue(LLVMGetCalledValue(Src)); LLVMBasicBlockRef Then = DeclareBB(LLVMGetNormalDest(Src)); LLVMBasicBlockRef Unwind = DeclareBB(LLVMGetUnwindDest(Src)); - Dst = LLVMBuildInvoke2(Builder, FnTy, Fn, Args.data(), ArgCount, - Then, Unwind, Name); + Dst = LLVMBuildInvokeWithOperandBundles( + Builder, FnTy, Fn, Args.data(), ArgCount, Then, Unwind, + Bundles.data(), Bundles.size(), Name); CloneAttrs(Src, Dst); + for (auto Bundle : Bundles) + LLVMDisposeOperandBundle(Bundle); break; } case LLVMUnreachable: @@ -764,14 +774,25 @@ struct FunCloner { } case LLVMCall: { SmallVector Args; - int ArgCount = LLVMGetNumArgOperands(Src); - for (int i = 0; i < ArgCount; i++) + SmallVector Bundles; + unsigned ArgCount = LLVMGetNumArgOperands(Src); + for (unsigned i = 0; i < ArgCount; ++i) Args.push_back(CloneValue(LLVMGetOperand(Src, i))); + unsigned BundleCount = LLVMGetNumOperandBundles(Src); + for (unsigned i = 0; i < BundleCount; ++i) { + auto Bundle = LLVMGetOperandBundleAtIndex(Src, i); + Bundles.push_back(CloneOB(Bundle)); + LLVMDisposeOperandBundle(Bundle); + } LLVMTypeRef FnTy = CloneType(LLVMGetCalledFunctionType(Src)); LLVMValueRef Fn = CloneValue(LLVMGetCalledValue(Src)); - Dst = LLVMBuildCall2(Builder, FnTy, Fn, Args.data(), ArgCount, Name); + Dst = LLVMBuildCallWithOperandBundles(Builder, FnTy, Fn, Args.data(), + ArgCount, Bundles.data(), + Bundles.size(), Name); LLVMSetTailCallKind(Dst, LLVMGetTailCallKind(Src)); CloneAttrs(Src, Dst); + for (auto Bundle : Bundles) + LLVMDisposeOperandBundle(Bundle); break; } case LLVMResume: { @@ -935,6 +956,17 @@ struct FunCloner { return VMap[Src] = Dst; } + LLVMOperandBundleRef CloneOB(LLVMOperandBundleRef Src) { + size_t TagLen; + const char *Tag = LLVMGetOperandBundleTag(Src, &TagLen); + + SmallVector Args; + for (unsigned i = 0, n = LLVMGetNumOperandBundleArgs(Src); i != n; ++i) + Args.push_back(CloneValue(LLVMGetOperandBundleArgAtIndex(Src, i))); + + return LLVMCreateOperandBundle(Tag, TagLen, Args.data(), Args.size()); + } + LLVMBasicBlockRef DeclareBB(LLVMBasicBlockRef Src) { // Check if this is something we already computed. { diff --git a/llvm/tools/llvm-cov/CodeCoverage.cpp b/llvm/tools/llvm-cov/CodeCoverage.cpp index b5d763d8643cd7d84c03431be60e984da26634a2..c1e3e2c4d1e2941cce5f043a727d63a697c680a5 100644 --- a/llvm/tools/llvm-cov/CodeCoverage.cpp +++ b/llvm/tools/llvm-cov/CodeCoverage.cpp @@ -502,7 +502,7 @@ void CodeCoverageTool::remapPathNames(const CoverageMapping &Coverage) { SmallString<128> NativeFilename; sys::path::native(Filename, NativeFilename); sys::path::remove_dots(NativeFilename, true); - if (NativeFilename.startswith(RemapFrom)) { + if (NativeFilename.starts_with(RemapFrom)) { RemappedFilenames[Filename] = RemapTo + NativeFilename.substr(RemapFrom.size()).str(); } diff --git a/llvm/tools/llvm-cov/CoverageReport.cpp b/llvm/tools/llvm-cov/CoverageReport.cpp index 060733b2d5d6ddf30044e5cab3b8631967cb661e..eae2653b85c80ba2e2c5209b90c36379ae958212 100644 --- a/llvm/tools/llvm-cov/CoverageReport.cpp +++ b/llvm/tools/llvm-cov/CoverageReport.cpp @@ -211,7 +211,7 @@ void CoverageReport::render(const FileCoverageSummary &File, sys::path::native(FileName); // remove_dots will remove trailing slash, so we need to check before it. - auto IsDir = FileName.endswith(sys::path::get_separator()); + auto IsDir = FileName.ends_with(sys::path::get_separator()); sys::path::remove_dots(FileName, /*remove_dot_dot=*/true); if (IsDir) FileName += sys::path::get_separator(); diff --git a/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp b/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp index 79a0494815c2b6a22cbb995974547ab94cee0bf4..d0a2e44be2521d16d8f9809fb4a2b5ac9c360188 100644 --- a/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp +++ b/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp @@ -639,7 +639,7 @@ struct CoveragePrinterHTMLDirectory::Reporter : public DirectoryCoverageReport { sys::path::native(LinkTextStr); // remove_dots will remove trailing slash, so we need to check before it. - auto IsDir = LinkTextStr.endswith(sys::path::get_separator()); + auto IsDir = LinkTextStr.ends_with(sys::path::get_separator()); sys::path::remove_dots(LinkTextStr, /*remove_dot_dot=*/true); SmallString<128> LinkTargetStr(LinkTextStr); diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h index 38111519a2c8987c152eb14b56e9174aa0d2b610..0d08febae20cb3e91b56290e00567f686a7ae660 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h @@ -70,6 +70,9 @@ struct BenchmarkKey { // An opaque configuration, that can be used to separate several benchmarks of // the same instruction under different configurations. std::string Config; + // The address that the snippet should be loaded in at if the execution mode + // being used supports it. + intptr_t SnippetAddress = 0; }; struct BenchmarkMeasure { diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index dd7c1a315531706201578a6b95bf240611c0d617..6c34446e8d6632943ecbf4c695dce1be87878d3f 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -44,7 +44,14 @@ #define GLIBC_INITS_RSEQ #endif #endif + +// Before kernel 4.17, Linux did not support MAP_FIXED_NOREPLACE, so if it is +// not available, simplfy define it as MAP_FIXED which performs the same +// function but does not guarantee existing mappings won't get clobbered. +#ifndef MAP_FIXED_NOREPLACE +#define MAP_FIXED_NOREPLACE MAP_FIXED #endif +#endif // __linux__ namespace llvm { namespace exegesis { @@ -416,9 +423,17 @@ private: #endif // GLIBC_INITS_RSEQ size_t FunctionDataCopySize = this->Function.FunctionBytes.size(); + void *MapAddress = NULL; + int MapFlags = MAP_PRIVATE | MAP_ANONYMOUS; + + if (Key.SnippetAddress != 0) { + MapAddress = reinterpret_cast(Key.SnippetAddress); + MapFlags |= MAP_FIXED_NOREPLACE; + } + char *FunctionDataCopy = - (char *)mmap(NULL, FunctionDataCopySize, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + (char *)mmap(MapAddress, FunctionDataCopySize, PROT_READ | PROT_WRITE, + MapFlags, 0, 0); if ((intptr_t)FunctionDataCopy == -1) exit(ChildProcessExitCodeE::FunctionDataMappingFailed); @@ -557,7 +572,7 @@ BenchmarkRunner::createFunctionExecutor( llvm_unreachable("ExecutionMode is outside expected range"); } -Expected BenchmarkRunner::runConfiguration( +std::pair BenchmarkRunner::runConfiguration( RunnableConfiguration &&RC, const std::optional &DumpFile) const { Benchmark &InstrBenchmark = RC.InstrBenchmark; @@ -568,8 +583,7 @@ Expected BenchmarkRunner::runConfiguration( auto ObjectFilePath = writeObjectFile(ObjectFile.getBinary()->getData(), *DumpFile); if (Error E = ObjectFilePath.takeError()) { - InstrBenchmark.Error = toString(std::move(E)); - return std::move(InstrBenchmark); + return {std::move(E), std::move(InstrBenchmark)}; } outs() << "Check generated assembly with: /usr/bin/objdump -d " << *ObjectFilePath << "\n"; @@ -577,20 +591,17 @@ Expected BenchmarkRunner::runConfiguration( if (BenchmarkPhaseSelector < BenchmarkPhaseSelectorE::Measure) { InstrBenchmark.Error = "actual measurements skipped."; - return std::move(InstrBenchmark); + return {Error::success(), std::move(InstrBenchmark)}; } Expected> Executor = createFunctionExecutor(std::move(ObjectFile), RC.InstrBenchmark.Key); if (!Executor) - return Executor.takeError(); + return {Executor.takeError(), std::move(InstrBenchmark)}; auto NewMeasurements = runMeasurements(**Executor); if (Error E = NewMeasurements.takeError()) { - if (!E.isA()) - return std::move(E); - InstrBenchmark.Error = toString(std::move(E)); - return std::move(InstrBenchmark); + return {std::move(E), std::move(InstrBenchmark)}; } assert(InstrBenchmark.NumRepetitions > 0 && "invalid NumRepetitions"); for (BenchmarkMeasure &BM : *NewMeasurements) { @@ -603,7 +614,7 @@ Expected BenchmarkRunner::runConfiguration( } InstrBenchmark.Measurements = std::move(*NewMeasurements); - return std::move(InstrBenchmark); + return {Error::success(), std::move(InstrBenchmark)}; } Expected diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h index 24f2086289408836fcfc8ee6fa121cf70730673b..2c48d07e37ca9ff14847b4fe06fee5139ccaa766 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h @@ -65,7 +65,7 @@ public: unsigned NumRepetitions, unsigned LoopUnrollFactor, const SnippetRepetitor &Repetitor) const; - Expected + std::pair runConfiguration(RunnableConfiguration &&RC, const std::optional &DumpFile) const; diff --git a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp index 753efed138163311fd6f514ce9cc3471ee6c9efc..eda450579a5838c36da535911cd1184634d9791a 100644 --- a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp @@ -21,11 +21,13 @@ namespace exegesis { LatencyBenchmarkRunner::LatencyBenchmarkRunner( const LLVMState &State, Benchmark::ModeE Mode, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, - Benchmark::ResultAggregationModeE ResultAgg, ExecutionModeE ExecutionMode) + Benchmark::ResultAggregationModeE ResultAgg, ExecutionModeE ExecutionMode, + unsigned BenchmarkRepeatCount) : BenchmarkRunner(State, Mode, BenchmarkPhaseSelector, ExecutionMode) { assert((Mode == Benchmark::Latency || Mode == Benchmark::InverseThroughput) && "invalid mode"); ResultAggMode = ResultAgg; + NumMeasurements = BenchmarkRepeatCount; } LatencyBenchmarkRunner::~LatencyBenchmarkRunner() = default; @@ -68,7 +70,6 @@ Expected> LatencyBenchmarkRunner::runMeasurements( // Cycle measurements include some overhead from the kernel. Repeat the // measure several times and return the aggregated value, as specified by // ResultAggMode. - constexpr const int NumMeasurements = 30; llvm::SmallVector AccumulatedValues; double MinVariance = std::numeric_limits::infinity(); const char *CounterName = State.getPfmCounters().CycleCounter; diff --git a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h index 34b912f0abded7cbdb1f9839634fac3b83c888a2..fc159d7d9b5e984541da448d9c577aee1be92dc0 100644 --- a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h +++ b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h @@ -24,7 +24,8 @@ public: LatencyBenchmarkRunner(const LLVMState &State, Benchmark::ModeE Mode, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, Benchmark::ResultAggregationModeE ResultAggMode, - ExecutionModeE ExecutionMode); + ExecutionModeE ExecutionMode, + unsigned BenchmarkRepeatCount); ~LatencyBenchmarkRunner() override; private: @@ -32,6 +33,7 @@ private: runMeasurements(const FunctionExecutor &Executor) const override; Benchmark::ResultAggregationModeE ResultAggMode; + unsigned NumMeasurements; }; } // namespace exegesis } // namespace llvm diff --git a/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp b/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp index d85a9f190655a5b45534ebbaeda0660604cb7436..286c606cd6fb19b83769f7cb334f99e22854f7f0 100644 --- a/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp +++ b/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp @@ -131,6 +131,19 @@ public: Result->Key.MemoryMappings.push_back(std::move(MemMap)); return; } + if (CommentText.consume_front("SNIPPET-ADDRESS")) { + // LLVM-EXEGESIS-SNIPPET-ADDRESS

+ if (!to_integer(CommentText.trim(), Result->Key.SnippetAddress, + 16)) { + errs() << "invalid comment 'LLVM-EXEGESIS-SNIPPET-ADDRESS " + << CommentText + << "', expected
to contain a valid integer in " + "hexadecimal format"; + ++InvalidComments; + return; + } + return; + } } unsigned numInvalidComments() const { return InvalidComments; } diff --git a/llvm/tools/llvm-exegesis/lib/Target.cpp b/llvm/tools/llvm-exegesis/lib/Target.cpp index aed093548f1580c24f1e18689280860a62d4e882..23c80e5b98953a12da0fd35dfc3e06e450fe9520 100644 --- a/llvm/tools/llvm-exegesis/lib/Target.cpp +++ b/llvm/tools/llvm-exegesis/lib/Target.cpp @@ -79,6 +79,7 @@ ExegesisTarget::createBenchmarkRunner( Benchmark::ModeE Mode, const LLVMState &State, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, BenchmarkRunner::ExecutionModeE ExecutionMode, + unsigned BenchmarkRepeatCount, Benchmark::ResultAggregationModeE ResultAggMode) const { PfmCountersInfo PfmCounters = State.getPfmCounters(); switch (Mode) { @@ -101,7 +102,8 @@ ExegesisTarget::createBenchmarkRunner( "the kernel for real event counts.")); } return createLatencyBenchmarkRunner(State, Mode, BenchmarkPhaseSelector, - ResultAggMode, ExecutionMode); + ResultAggMode, ExecutionMode, + BenchmarkRepeatCount); case Benchmark::Uops: if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure && !PfmCounters.UopsCounter && !PfmCounters.IssueCounters) @@ -130,9 +132,11 @@ std::unique_ptr ExegesisTarget::createLatencyBenchmarkRunner( const LLVMState &State, Benchmark::ModeE Mode, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, Benchmark::ResultAggregationModeE ResultAggMode, - BenchmarkRunner::ExecutionModeE ExecutionMode) const { + BenchmarkRunner::ExecutionModeE ExecutionMode, + unsigned BenchmarkRepeatCount) const { return std::make_unique( - State, Mode, BenchmarkPhaseSelector, ResultAggMode, ExecutionMode); + State, Mode, BenchmarkPhaseSelector, ResultAggMode, ExecutionMode, + BenchmarkRepeatCount); } std::unique_ptr ExegesisTarget::createUopsBenchmarkRunner( diff --git a/llvm/tools/llvm-exegesis/lib/Target.h b/llvm/tools/llvm-exegesis/lib/Target.h index 6de5b3c1065f1aab5abcc023e332437383771c0d..c37dd8b70821623d46a0135812146f2f06aadc96 100644 --- a/llvm/tools/llvm-exegesis/lib/Target.h +++ b/llvm/tools/llvm-exegesis/lib/Target.h @@ -262,6 +262,7 @@ public: Benchmark::ModeE Mode, const LLVMState &State, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, BenchmarkRunner::ExecutionModeE ExecutionMode, + unsigned BenchmarkRepeatCount, Benchmark::ResultAggregationModeE ResultAggMode = Benchmark::Min) const; // Returns the ExegesisTarget for the given triple or nullptr if the target @@ -305,7 +306,8 @@ private: const LLVMState &State, Benchmark::ModeE Mode, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, Benchmark::ResultAggregationModeE ResultAggMode, - BenchmarkRunner::ExecutionModeE ExecutionMode) const; + BenchmarkRunner::ExecutionModeE ExecutionMode, + unsigned BenchmarkRepeatCount) const; std::unique_ptr virtual createUopsBenchmarkRunner( const LLVMState &State, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, Benchmark::ResultAggregationModeE ResultAggMode, diff --git a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp index 261335a817d0674644d4f9454417688cc3fde70c..148891a18246f630cf3e516e4066a150ee029301 100644 --- a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp +++ b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp @@ -262,6 +262,12 @@ static cl::opt ExecutionMode( "allows for the use of memory annotations")), cl::init(BenchmarkRunner::ExecutionModeE::InProcess)); +static cl::opt BenchmarkRepeatCount( + "benchmark-repeat-count", + cl::desc("The number of times to repeat measurements on the benchmark k " + "before aggregating the results"), + cl::cat(BenchmarkOptions), cl::init(30)); + static ExitOnError ExitOnErr("llvm-exegesis error: "); // Helper function that logs the error(s) and exits. @@ -404,8 +410,18 @@ static void runBenchmarkConfigurations( std::optional DumpFile; if (DumpObjectToDisk.getNumOccurrences()) DumpFile = DumpObjectToDisk; - AllResults.emplace_back( - ExitOnErr(Runner.runConfiguration(std::move(RC), DumpFile))); + auto [Err, InstrBenchmark] = + Runner.runConfiguration(std::move(RC), DumpFile); + if (Err) { + // Errors from executing the snippets are fine. + // All other errors are a framework issue and should fail. + if (!Err.isA()) { + llvm::errs() << "llvm-exegesis error: " << toString(std::move(Err)); + exit(1); + } + InstrBenchmark.Error = toString(std::move(Err)); + } + AllResults.push_back(std::move(InstrBenchmark)); } Benchmark &Result = AllResults.front(); @@ -485,7 +501,7 @@ void benchmarkMain() { const std::unique_ptr Runner = ExitOnErr(State.getExegesisTarget().createBenchmarkRunner( BenchmarkMode, State, BenchmarkPhaseSelector, ExecutionMode, - ResultAggMode)); + BenchmarkRepeatCount, ResultAggMode)); if (!Runner) { ExitWithError("cannot create benchmark runner"); } @@ -533,8 +549,10 @@ void benchmarkMain() { for (const auto &Configuration : Configurations) { if (ExecutionMode != BenchmarkRunner::ExecutionModeE::SubProcess && (Configuration.Key.MemoryMappings.size() != 0 || - Configuration.Key.MemoryValues.size() != 0)) - ExitWithError("Memory annotations are only supported in subprocess " + Configuration.Key.MemoryValues.size() != 0 || + Configuration.Key.SnippetAddress != 0)) + ExitWithError("Memory and snippet address annotations are only " + "supported in subprocess " "execution mode"); } } diff --git a/llvm/tools/llvm-ml/llvm-ml.cpp b/llvm/tools/llvm-ml/llvm-ml.cpp index 4d6bd90d13ad52436df1fa32e832c266cee9ae70..1c713304e4ea17d3b5657ec4967a154c94836ea6 100644 --- a/llvm/tools/llvm-ml/llvm-ml.cpp +++ b/llvm/tools/llvm-ml/llvm-ml.cpp @@ -83,7 +83,7 @@ static Triple GetTriple(StringRef ProgName, opt::InputArgList &Args) { StringRef DefaultBitness = "32"; SmallString<255> Program = ProgName; sys::path::replace_extension(Program, ""); - if (Program.endswith("ml64")) + if (Program.ends_with("ml64")) DefaultBitness = "64"; StringRef TripleName = diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp index d6d0ea35044ab30b73188ec35cbbb240deeeb0f0..a9a31ae6c4740b9c476e81059150e808761e302b 100644 --- a/llvm/tools/llvm-readobj/ELFDumper.cpp +++ b/llvm/tools/llvm-readobj/ELFDumper.cpp @@ -2027,6 +2027,18 @@ template void ELFDumper::parseDynamicTable() { uint64_t StringTableSize = 0; std::optional DynSymFromTable; for (const Elf_Dyn &Dyn : dynamic_table()) { + if (Obj.getHeader().e_machine == EM_AARCH64) { + switch (Dyn.d_tag) { + case ELF::DT_AARCH64_AUTH_RELRSZ: + DynRelrRegion.Size = Dyn.getVal(); + DynRelrRegion.SizePrintName = "DT_AARCH64_AUTH_RELRSZ value"; + continue; + case ELF::DT_AARCH64_AUTH_RELRENT: + DynRelrRegion.EntSize = Dyn.getVal(); + DynRelrRegion.EntSizePrintName = "DT_AARCH64_AUTH_RELRENT value"; + continue; + } + } switch (Dyn.d_tag) { case ELF::DT_HASH: HashTable = reinterpret_cast( @@ -2090,10 +2102,12 @@ template void ELFDumper::parseDynamicTable() { break; case ELF::DT_RELR: case ELF::DT_ANDROID_RELR: + case ELF::DT_AARCH64_AUTH_RELR: DynRelrRegion.Addr = toMappedAddr(Dyn.getTag(), Dyn.getPtr()); break; case ELF::DT_RELRSZ: case ELF::DT_ANDROID_RELRSZ: + case ELF::DT_AARCH64_AUTH_RELRSZ: DynRelrRegion.Size = Dyn.getVal(); DynRelrRegion.SizePrintName = Dyn.d_tag == ELF::DT_RELRSZ ? "DT_RELRSZ value" @@ -2101,6 +2115,7 @@ template void ELFDumper::parseDynamicTable() { break; case ELF::DT_RELRENT: case ELF::DT_ANDROID_RELRENT: + case ELF::DT_AARCH64_AUTH_RELRENT: DynRelrRegion.EntSize = Dyn.getVal(); DynRelrRegion.EntSizePrintName = Dyn.d_tag == ELF::DT_RELRENT ? "DT_RELRENT value" @@ -2467,6 +2482,8 @@ std::string ELFDumper::getDynamicEntry(uint64_t Type, case DT_PREINIT_ARRAYSZ: case DT_RELRSZ: case DT_RELRENT: + case DT_AARCH64_AUTH_RELRSZ: + case DT_AARCH64_AUTH_RELRENT: case DT_ANDROID_RELSZ: case DT_ANDROID_RELASZ: return std::to_string(Value) + " (bytes)"; @@ -2564,7 +2581,7 @@ template void ELFDumper::printNeededLibraries() { llvm::sort(Libs); for (StringRef L : Libs) - W.startLine() << L << "\n"; + W.printString(L); } template @@ -3799,9 +3816,12 @@ void GNUELFDumper::printRelRelaReloc(const Relocation &R, } template -static void printRelocHeaderFields(formatted_raw_ostream &OS, unsigned SType) { +static void printRelocHeaderFields(formatted_raw_ostream &OS, unsigned SType, + const typename ELFT::Ehdr &EHeader) { bool IsRela = SType == ELF::SHT_RELA || SType == ELF::SHT_ANDROID_RELA; - bool IsRelr = SType == ELF::SHT_RELR || SType == ELF::SHT_ANDROID_RELR; + bool IsRelr = + SType == ELF::SHT_RELR || SType == ELF::SHT_ANDROID_RELR || + (EHeader.e_machine == EM_AARCH64 && SType == ELF::SHT_AARCH64_AUTH_RELR); if (ELFT::Is64Bits) OS << " "; else @@ -3826,15 +3846,18 @@ void GNUELFDumper::printDynamicRelocHeader(unsigned Type, StringRef Name, uint64_t Offset = Reg.Addr - this->Obj.base(); OS << "\n'" << Name.str().c_str() << "' relocation section at offset 0x" << utohexstr(Offset, /*LowerCase=*/true) << " contains " << Reg.Size << " bytes:\n"; - printRelocHeaderFields(OS, Type); + printRelocHeaderFields(OS, Type, this->Obj.getHeader()); } template -static bool isRelocationSec(const typename ELFT::Shdr &Sec) { +static bool isRelocationSec(const typename ELFT::Shdr &Sec, + const typename ELFT::Ehdr &EHeader) { return Sec.sh_type == ELF::SHT_REL || Sec.sh_type == ELF::SHT_RELA || Sec.sh_type == ELF::SHT_RELR || Sec.sh_type == ELF::SHT_ANDROID_REL || Sec.sh_type == ELF::SHT_ANDROID_RELA || - Sec.sh_type == ELF::SHT_ANDROID_RELR; + Sec.sh_type == ELF::SHT_ANDROID_RELR || + (EHeader.e_machine == EM_AARCH64 && + Sec.sh_type == ELF::SHT_AARCH64_AUTH_RELR); } template void GNUELFDumper::printRelocations() { @@ -3850,8 +3873,10 @@ template void GNUELFDumper::printRelocations() { return RelasOrErr->size(); } - if (!opts::RawRelr && (Sec.sh_type == ELF::SHT_RELR || - Sec.sh_type == ELF::SHT_ANDROID_RELR)) { + if (!opts::RawRelr && + (Sec.sh_type == ELF::SHT_RELR || Sec.sh_type == ELF::SHT_ANDROID_RELR || + (this->Obj.getHeader().e_machine == EM_AARCH64 && + Sec.sh_type == ELF::SHT_AARCH64_AUTH_RELR))) { Expected RelrsOrErr = this->Obj.relrs(Sec); if (!RelrsOrErr) return RelrsOrErr.takeError(); @@ -3863,7 +3888,7 @@ template void GNUELFDumper::printRelocations() { bool HasRelocSections = false; for (const Elf_Shdr &Sec : cantFail(this->Obj.sections())) { - if (!isRelocationSec(Sec)) + if (!isRelocationSec(Sec, this->Obj.getHeader())) continue; HasRelocSections = true; @@ -3880,7 +3905,7 @@ template void GNUELFDumper::printRelocations() { OS << "\nRelocation section '" << Name << "' at offset 0x" << utohexstr(Offset, /*LowerCase=*/true) << " contains " << EntriesNum << " entries:\n"; - printRelocHeaderFields(OS, Sec.sh_type); + printRelocHeaderFields(OS, Sec.sh_type, this->Obj.getHeader()); this->printRelocationsHelper(Sec); } if (!HasRelocSections) @@ -5312,6 +5337,31 @@ static bool printAndroidNote(raw_ostream &OS, uint32_t NoteType, return true; } +template +static bool printAArch64Note(raw_ostream &OS, uint32_t NoteType, + ArrayRef Desc) { + if (NoteType != NT_ARM_TYPE_PAUTH_ABI_TAG) + return false; + + OS << " AArch64 PAuth ABI tag: "; + if (Desc.size() < 16) { + OS << format("", Desc.size()); + return false; + } + + uint64_t Platform = + support::endian::read64(Desc.data() + 0); + uint64_t Version = + support::endian::read64(Desc.data() + 8); + OS << format("platform 0x%" PRIx64 ", version 0x%" PRIx64, Platform, Version); + + if (Desc.size() > 16) + OS << ", additional info 0x" + << toHex(ArrayRef(Desc.data() + 16, Desc.size() - 16)); + + return true; +} + template void GNUELFDumper::printMemtag( const ArrayRef> DynamicEntries, @@ -5711,6 +5761,10 @@ const NoteType AndroidNoteTypes[] = { "NT_ANDROID_TYPE_MEMTAG (Android memory tagging information)"}, }; +const NoteType ARMNoteTypes[] = { + {ELF::NT_ARM_TYPE_PAUTH_ABI_TAG, "NT_ARM_TYPE_PAUTH_ABI_TAG"}, +}; + const NoteType CoreNoteTypes[] = { {ELF::NT_PRSTATUS, "NT_PRSTATUS (prstatus structure)"}, {ELF::NT_FPREGSET, "NT_FPREGSET (floating point registers)"}, @@ -5829,6 +5883,8 @@ StringRef getNoteTypeName(const typename ELFT::Note &Note, unsigned ELFType) { return FindNote(LLVMOMPOFFLOADNoteTypes); if (Name == "Android") return FindNote(AndroidNoteTypes); + if (Name == "ARM") + return FindNote(ARMNoteTypes); if (ELFType == ELF::ET_CORE) return FindNote(CoreNoteTypes); @@ -5984,6 +6040,9 @@ template void GNUELFDumper::printNotes() { } else if (Name == "Android") { if (printAndroidNote(OS, Type, Descriptor)) return Error::success(); + } else if (Name == "ARM") { + if (printAArch64Note(OS, Type, Descriptor)) + return Error::success(); } if (!Descriptor.empty()) { OS << " description data:"; @@ -6176,11 +6235,13 @@ void ELFDumper::forEachRelocationDo( toString(std::move(E))); }; - // SHT_RELR/SHT_ANDROID_RELR sections do not have an associated symbol table. - // For them we should not treat the value of the sh_link field as an index of - // a symbol table. + // SHT_RELR/SHT_ANDROID_RELR/SHT_AARCH64_AUTH_RELR sections do not have an + // associated symbol table. For them we should not treat the value of the + // sh_link field as an index of a symbol table. const Elf_Shdr *SymTab; - if (Sec.sh_type != ELF::SHT_RELR && Sec.sh_type != ELF::SHT_ANDROID_RELR) { + if (Sec.sh_type != ELF::SHT_RELR && Sec.sh_type != ELF::SHT_ANDROID_RELR && + !(Obj.getHeader().e_machine == EM_AARCH64 && + Sec.sh_type == ELF::SHT_AARCH64_AUTH_RELR)) { Expected SymTabOrErr = Obj.getSection(Sec.sh_link); if (!SymTabOrErr) { Warn(SymTabOrErr.takeError(), "unable to locate a symbol table for"); @@ -6208,6 +6269,10 @@ void ELFDumper::forEachRelocationDo( Warn(RangeOrErr.takeError()); } break; + case ELF::SHT_AARCH64_AUTH_RELR: + if (Obj.getHeader().e_machine != EM_AARCH64) + break; + [[fallthrough]]; case ELF::SHT_RELR: case ELF::SHT_ANDROID_RELR: { Expected RangeOrErr = Obj.relrs(Sec); @@ -6904,7 +6969,7 @@ template void LLVMELFDumper::printRelocations() { ListScope D(W, "Relocations"); for (const Elf_Shdr &Sec : cantFail(this->Obj.sections())) { - if (!isRelocationSec(Sec)) + if (!isRelocationSec(Sec, this->Obj.getHeader())) continue; StringRef Name = this->getPrintableSectionName(Sec); @@ -7545,6 +7610,29 @@ static bool printAndroidNoteLLVMStyle(uint32_t NoteType, ArrayRef Desc, return true; } +template +static bool printAarch64NoteLLVMStyle(uint32_t NoteType, ArrayRef Desc, + ScopedPrinter &W) { + if (NoteType != NT_ARM_TYPE_PAUTH_ABI_TAG) + return false; + + if (Desc.size() < 16) + return false; + + uint64_t platform = + support::endian::read64(Desc.data() + 0); + uint64_t version = + support::endian::read64(Desc.data() + 8); + W.printNumber("Platform", platform); + W.printNumber("Version", version); + + if (Desc.size() > 16) + W.printString("Additional info", + toHex(ArrayRef(Desc.data() + 16, Desc.size() - 16))); + + return true; +} + template void LLVMELFDumper::printMemtag( const ArrayRef> DynamicEntries, @@ -7681,6 +7769,9 @@ template void LLVMELFDumper::printNotes() { } else if (Name == "Android") { if (printAndroidNoteLLVMStyle(Type, Descriptor, W)) return Error::success(); + } else if (Name == "ARM") { + if (printAarch64NoteLLVMStyle(Type, Descriptor, W)) + return Error::success(); } if (!Descriptor.empty()) { W.printBinaryBlock("Description data", Descriptor); diff --git a/llvm/tools/llvm-readtapi/llvm-readtapi.cpp b/llvm/tools/llvm-readtapi/llvm-readtapi.cpp index cb2b36072a552b1f755c1b3f60279113c57bd1a5..64eff70dbfc75d25aa4ddf6681369bf4142dfdd8 100644 --- a/llvm/tools/llvm-readtapi/llvm-readtapi.cpp +++ b/llvm/tools/llvm-readtapi/llvm-readtapi.cpp @@ -56,30 +56,31 @@ public: } }; +struct Context { + std::vector Inputs; + std::unique_ptr OutStream; + FileType WriteFT = FileType::TBD_V5; + bool Compact = false; + Architecture Arch = AK_unknown; +}; + // Use unique exit code to differentiate failures not directly caused from // TextAPI operations. This is used for wrapping `compare` operations in // automation and scripting. const int NON_TAPI_EXIT_CODE = 2; const std::string TOOLNAME = "llvm-readtapi"; ExitOnError ExitOnErr; +} // anonymous namespace // Handle error reporting in cases where `ExitOnError` is not used. -void reportError(Twine Message, int ExitCode = EXIT_FAILURE) { +static void reportError(Twine Message, int ExitCode = EXIT_FAILURE) { errs() << TOOLNAME << ": error: " << Message << "\n"; errs().flush(); exit(ExitCode); } -struct Context { - std::vector Inputs; - std::unique_ptr OutStream; - FileType WriteFT = FileType::TBD_V5; - bool Compact = false; - Architecture Arch = AK_unknown; -}; - -std::unique_ptr getInterfaceFile(const StringRef Filename, - bool ResetBanner = true) { +static std::unique_ptr +getInterfaceFile(const StringRef Filename, bool ResetBanner = true) { ExitOnErr.setBanner(TOOLNAME + ": error: '" + Filename.str() + "' "); ErrorOr> BufferOrErr = MemoryBuffer::getFile(Filename); @@ -94,7 +95,7 @@ std::unique_ptr getInterfaceFile(const StringRef Filename, return std::move(*IF); } -bool handleCompareAction(const Context &Ctx) { +static bool handleCompareAction(const Context &Ctx) { if (Ctx.Inputs.size() != 2) reportError("compare only supports two input files", /*ExitCode=*/NON_TAPI_EXIT_CODE); @@ -109,8 +110,8 @@ bool handleCompareAction(const Context &Ctx) { return DiffEngine(LeftIF.get(), RightIF.get()).compareFiles(OS); } -bool handleWriteAction(const Context &Ctx, - std::unique_ptr Out = nullptr) { +static bool handleWriteAction(const Context &Ctx, + std::unique_ptr Out = nullptr) { if (!Out) { if (Ctx.Inputs.size() != 1) reportError("write only supports one input file"); @@ -121,7 +122,7 @@ bool handleWriteAction(const Context &Ctx, return EXIT_SUCCESS; } -bool handleMergeAction(const Context &Ctx) { +static bool handleMergeAction(const Context &Ctx) { if (Ctx.Inputs.size() < 2) reportError("merge requires at least two input files"); @@ -144,8 +145,8 @@ bool handleMergeAction(const Context &Ctx) { using IFOperation = std::function>( const llvm::MachO::InterfaceFile &, Architecture)>; -bool handleSingleFileAction(const Context &Ctx, const StringRef Action, - IFOperation act) { +static bool handleSingleFileAction(const Context &Ctx, const StringRef Action, + IFOperation act) { if (Ctx.Inputs.size() != 1) reportError(Action + " only supports one input file"); if (Ctx.Arch == AK_unknown) @@ -159,8 +160,6 @@ bool handleSingleFileAction(const Context &Ctx, const StringRef Action, return handleWriteAction(Ctx, std::move(*OutIF)); } -} // anonymous namespace - int main(int Argc, char **Argv) { InitLLVM X(Argc, Argv); BumpPtrAllocator A; diff --git a/llvm/tools/llvm-shlib/CMakeLists.txt b/llvm/tools/llvm-shlib/CMakeLists.txt index 8ace190b28f74472adb18781aa5f9ddf06a82e0b..64d6f631ffadd53fab497fafd9522f9cb4953dd1 100644 --- a/llvm/tools/llvm-shlib/CMakeLists.txt +++ b/llvm/tools/llvm-shlib/CMakeLists.txt @@ -14,6 +14,9 @@ if(LLVM_BUILD_LLVM_DYLIB) if(MSVC) message(FATAL_ERROR "Generating libLLVM is not supported on MSVC") endif() + if(ZOS) + message(FATAL_ERROR "Generating libLLVM is not supported on z/OS") + endif() llvm_map_components_to_libnames(LIB_NAMES ${LLVM_DYLIB_COMPONENTS}) @@ -33,16 +36,9 @@ if(LLVM_BUILD_LLVM_DYLIB) add_llvm_library(LLVM SHARED DISABLE_LLVM_LINK_LLVM_DYLIB SONAME ${INSTALL_WITH_TOOLCHAIN} ${SOURCES}) list(REMOVE_DUPLICATES LIB_NAMES) - if((MINGW) OR (HAIKU) - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "GNU") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "NetBSD") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "OpenBSD") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "DragonFly") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "Fuchsia") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "Android") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "SunOS")) # FIXME: It should be "GNU ld for elf" + if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") + set(LIB_NAMES -Wl,-all_load ${LIB_NAMES}) + else() configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/simple_version_script.map.in ${LLVM_LIBRARY_DIR}/tools/llvm-shlib/simple_version_script.map) @@ -60,8 +56,6 @@ if(LLVM_BUILD_LLVM_DYLIB) # inside and outside libLLVM.so. target_link_options(LLVM PRIVATE LINKER:-Bsymbolic-functions) endif() - elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") - set(LIB_NAMES -Wl,-all_load ${LIB_NAMES}) endif() target_link_libraries(LLVM PRIVATE ${LIB_NAMES}) diff --git a/llvm/tools/obj2yaml/xcoff2yaml.cpp b/llvm/tools/obj2yaml/xcoff2yaml.cpp index 882c410496012a1de46b759e77b6ad6d52b379f8..f7c2bae74798951586d82b14d718efc3f2e5ce90 100644 --- a/llvm/tools/obj2yaml/xcoff2yaml.cpp +++ b/llvm/tools/obj2yaml/xcoff2yaml.cpp @@ -25,10 +25,30 @@ class XCOFFDumper { template Error dumpSections(ArrayRef Sections); + // Dump auxiliary symbols. + Error dumpFileAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef); + Error dumpStatAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef); + Error dumpBlockAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef); + Error dumpDwarfAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef); + Error dumpAuxSyms(XCOFFYAML::Symbol &Sym, const XCOFFSymbolRef &SymbolEntRef); + void dumpFuncAuxSym(XCOFFYAML::Symbol &Sym, const uintptr_t AuxAddress); + void dumpExpAuxSym(XCOFFYAML::Symbol &Sym, const uintptr_t AuxAddress); + void dumpCscetAuxSym(XCOFFYAML::Symbol &Sym, + const object::XCOFFCsectAuxRef &AuxEntPtr); + public: XCOFFDumper(const object::XCOFFObjectFile &obj) : Obj(obj) {} Error dump(); XCOFFYAML::Object &getYAMLObj() { return YAMLObj; } + + template const T *getAuxEntPtr(uintptr_t AuxAddress) { + Obj.checkSymbolEntryPointer(AuxAddress); + return reinterpret_cast(AuxAddress); + } }; } // namespace @@ -106,6 +126,210 @@ Error XCOFFDumper::dumpSections(ArrayRef Sections) { return Error::success(); } +Error XCOFFDumper::dumpFileAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef) { + for (uint8_t I = 1; I <= Sym.NumberOfAuxEntries; ++I) { + uintptr_t AuxAddress = XCOFFObjectFile::getAdvancedSymbolEntryAddress( + SymbolEntRef.getEntryAddress(), I); + const XCOFFFileAuxEnt *FileAuxEntPtr = + getAuxEntPtr(AuxAddress); + auto FileNameOrError = Obj.getCFileName(FileAuxEntPtr); + if (!FileNameOrError) + return FileNameOrError.takeError(); + + XCOFFYAML::FileAuxEnt FileAuxSym; + FileAuxSym.FileNameOrString = FileNameOrError.get(); + FileAuxSym.FileStringType = FileAuxEntPtr->Type; + Sym.AuxEntries.push_back( + std::make_unique(FileAuxSym)); + } + return Error::success(); +} + +Error XCOFFDumper::dumpStatAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef) { + if (Sym.NumberOfAuxEntries != 1) { + uint32_t SymbolIndex = Obj.getSymbolIndex(SymbolEntRef.getEntryAddress()); + return createError("failed to parse symbol \"" + Sym.SymbolName + + "\" with index of " + Twine(SymbolIndex) + + ": expected 1 aux symbol for C_STAT, while got " + + Twine(static_cast(*Sym.NumberOfAuxEntries))); + } + + const XCOFFSectAuxEntForStat *AuxEntPtr = + getAuxEntPtr( + XCOFFObjectFile::getAdvancedSymbolEntryAddress( + SymbolEntRef.getEntryAddress(), 1)); + XCOFFYAML::SectAuxEntForStat StatAuxSym; + StatAuxSym.SectionLength = AuxEntPtr->SectionLength; + StatAuxSym.NumberOfLineNum = AuxEntPtr->NumberOfLineNum; + StatAuxSym.NumberOfRelocEnt = AuxEntPtr->NumberOfRelocEnt; + Sym.AuxEntries.push_back( + std::make_unique(StatAuxSym)); + return Error::success(); +} + +void XCOFFDumper::dumpFuncAuxSym(XCOFFYAML::Symbol &Sym, + const uintptr_t AuxAddress) { + XCOFFYAML::FunctionAuxEnt FunAuxSym; + + if (Obj.is64Bit()) { + const XCOFFFunctionAuxEnt64 *AuxEntPtr = + getAuxEntPtr(AuxAddress); + FunAuxSym.PtrToLineNum = AuxEntPtr->PtrToLineNum; + FunAuxSym.SizeOfFunction = AuxEntPtr->SizeOfFunction; + FunAuxSym.SymIdxOfNextBeyond = AuxEntPtr->SymIdxOfNextBeyond; + } else { + const XCOFFFunctionAuxEnt32 *AuxEntPtr = + getAuxEntPtr(AuxAddress); + FunAuxSym.OffsetToExceptionTbl = AuxEntPtr->OffsetToExceptionTbl; + FunAuxSym.PtrToLineNum = AuxEntPtr->PtrToLineNum; + FunAuxSym.SizeOfFunction = AuxEntPtr->SizeOfFunction; + FunAuxSym.SymIdxOfNextBeyond = AuxEntPtr->SymIdxOfNextBeyond; + } + + Sym.AuxEntries.push_back( + std::make_unique(FunAuxSym)); +} + +void XCOFFDumper::dumpExpAuxSym(XCOFFYAML::Symbol &Sym, + const uintptr_t AuxAddress) { + const XCOFFExceptionAuxEnt *AuxEntPtr = + getAuxEntPtr(AuxAddress); + XCOFFYAML::ExcpetionAuxEnt ExceptAuxSym; + ExceptAuxSym.OffsetToExceptionTbl = AuxEntPtr->OffsetToExceptionTbl; + ExceptAuxSym.SizeOfFunction = AuxEntPtr->SizeOfFunction; + ExceptAuxSym.SymIdxOfNextBeyond = AuxEntPtr->SymIdxOfNextBeyond; + Sym.AuxEntries.push_back( + std::make_unique(ExceptAuxSym)); +} + +void XCOFFDumper::dumpCscetAuxSym(XCOFFYAML::Symbol &Sym, + const object::XCOFFCsectAuxRef &AuxEntPtr) { + XCOFFYAML::CsectAuxEnt CsectAuxSym; + CsectAuxSym.ParameterHashIndex = AuxEntPtr.getParameterHashIndex(); + CsectAuxSym.TypeChkSectNum = AuxEntPtr.getTypeChkSectNum(); + CsectAuxSym.SymbolAlignmentAndType = AuxEntPtr.getSymbolAlignmentAndType(); + CsectAuxSym.StorageMappingClass = AuxEntPtr.getStorageMappingClass(); + + if (Obj.is64Bit()) { + CsectAuxSym.SectionOrLengthLo = + static_cast(AuxEntPtr.getSectionOrLength64()); + CsectAuxSym.SectionOrLengthHi = + static_cast(AuxEntPtr.getSectionOrLength64() >> 32); + } else { + CsectAuxSym.SectionOrLength = AuxEntPtr.getSectionOrLength32(); + CsectAuxSym.StabInfoIndex = AuxEntPtr.getStabInfoIndex32(); + CsectAuxSym.StabSectNum = AuxEntPtr.getStabSectNum32(); + } + + Sym.AuxEntries.push_back( + std::make_unique(CsectAuxSym)); +} + +Error XCOFFDumper::dumpAuxSyms(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef) { + auto ErrOrCsectAuxRef = SymbolEntRef.getXCOFFCsectAuxRef(); + if (!ErrOrCsectAuxRef) + return ErrOrCsectAuxRef.takeError(); + XCOFFCsectAuxRef CsectAuxRef = ErrOrCsectAuxRef.get(); + + for (uint8_t I = 1; I <= Sym.NumberOfAuxEntries; ++I) { + + if (I == Sym.NumberOfAuxEntries && !Obj.is64Bit()) { + dumpCscetAuxSym(Sym, CsectAuxRef); + return Error::success(); + } + + uintptr_t AuxAddress = XCOFFObjectFile::getAdvancedSymbolEntryAddress( + SymbolEntRef.getEntryAddress(), I); + + if (Obj.is64Bit()) { + XCOFF::SymbolAuxType Type = *Obj.getSymbolAuxType(AuxAddress); + if (Type == XCOFF::SymbolAuxType::AUX_CSECT) + dumpCscetAuxSym(Sym, CsectAuxRef); + else if (Type == XCOFF::SymbolAuxType::AUX_FCN) + dumpFuncAuxSym(Sym, AuxAddress); + else if (Type == XCOFF::SymbolAuxType::AUX_EXCEPT) + dumpExpAuxSym(Sym, AuxAddress); + else { + uint32_t SymbolIndex = + Obj.getSymbolIndex(SymbolEntRef.getEntryAddress()); + return createError("failed to parse symbol \"" + Sym.SymbolName + + "\" with index of " + Twine(SymbolIndex) + + ": invalid auxiliary symbol type: " + + Twine(static_cast(Type))); + } + + } else + dumpFuncAuxSym(Sym, AuxAddress); + } + + return Error::success(); +} + +Error XCOFFDumper::dumpBlockAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef) { + if (Sym.NumberOfAuxEntries != 1) { + uint32_t SymbolIndex = Obj.getSymbolIndex(SymbolEntRef.getEntryAddress()); + return createError( + "failed to parse symbol \"" + Sym.SymbolName + "\" with index of " + + Twine(SymbolIndex) + + ": expected 1 aux symbol for C_BLOCK or C_FCN, while got " + + Twine(static_cast(*Sym.NumberOfAuxEntries))); + } + + uintptr_t AuxAddress = XCOFFObjectFile::getAdvancedSymbolEntryAddress( + SymbolEntRef.getEntryAddress(), 1); + XCOFFYAML::BlockAuxEnt BlockAuxSym; + + if (Obj.is64Bit()) { + const XCOFFBlockAuxEnt64 *AuxEntPtr = + getAuxEntPtr(AuxAddress); + BlockAuxSym.LineNum = AuxEntPtr->LineNum; + } else { + const XCOFFBlockAuxEnt32 *AuxEntPtr = + getAuxEntPtr(AuxAddress); + BlockAuxSym.LineNumLo = AuxEntPtr->LineNumLo; + BlockAuxSym.LineNumHi = AuxEntPtr->LineNumHi; + } + + Sym.AuxEntries.push_back( + std::make_unique(BlockAuxSym)); + return Error::success(); +} + +Error XCOFFDumper::dumpDwarfAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef) { + if (Sym.NumberOfAuxEntries != 1) { + uint32_t SymbolIndex = Obj.getSymbolIndex(SymbolEntRef.getEntryAddress()); + return createError("failed to parse symbol \"" + Sym.SymbolName + + "\" with index of " + Twine(SymbolIndex) + + ": expected 1 aux symbol for C_DWARF, while got " + + Twine(static_cast(*Sym.NumberOfAuxEntries))); + } + + uintptr_t AuxAddress = XCOFFObjectFile::getAdvancedSymbolEntryAddress( + SymbolEntRef.getEntryAddress(), 1); + XCOFFYAML::SectAuxEntForDWARF DwarfAuxSym; + + if (Obj.is64Bit()) { + const XCOFFSectAuxEntForDWARF64 *AuxEntPtr = + getAuxEntPtr(AuxAddress); + DwarfAuxSym.LengthOfSectionPortion = AuxEntPtr->LengthOfSectionPortion; + DwarfAuxSym.NumberOfRelocEnt = AuxEntPtr->NumberOfRelocEnt; + } else { + const XCOFFSectAuxEntForDWARF32 *AuxEntPtr = + getAuxEntPtr(AuxAddress); + DwarfAuxSym.LengthOfSectionPortion = AuxEntPtr->LengthOfSectionPortion; + DwarfAuxSym.NumberOfRelocEnt = AuxEntPtr->NumberOfRelocEnt; + } + + Sym.AuxEntries.push_back( + std::make_unique(DwarfAuxSym)); + return Error::success(); +} + Error XCOFFDumper::dumpSymbols() { std::vector &Symbols = YAMLObj.Symbols; @@ -133,6 +357,36 @@ Error XCOFFDumper::dumpSymbols() { Sym.StorageClass = SymbolEntRef.getStorageClass(); Sym.NumberOfAuxEntries = SymbolEntRef.getNumberOfAuxEntries(); + if (Sym.NumberOfAuxEntries) { + switch (Sym.StorageClass) { + case XCOFF::C_FILE: + if (Error E = dumpFileAuxSym(Sym, SymbolEntRef)) + return E; + break; + case XCOFF::C_STAT: + if (Error E = dumpStatAuxSym(Sym, SymbolEntRef)) + return E; + break; + case XCOFF::C_EXT: + case XCOFF::C_WEAKEXT: + case XCOFF::C_HIDEXT: + if (Error E = dumpAuxSyms(Sym, SymbolEntRef)) + return E; + break; + case XCOFF::C_BLOCK: + case XCOFF::C_FCN: + if (Error E = dumpBlockAuxSym(Sym, SymbolEntRef)) + return E; + break; + case XCOFF::C_DWARF: + if (Error E = dumpDwarfAuxSym(Sym, SymbolEntRef)) + return E; + break; + default: + break; + } + } + Symbols.push_back(std::move(Sym)); } diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp index 5e5e5ce233f310acbe5445cfa2ecfaea7fba8c42..50b36dc7426777fd40ea1ea6e1aa74d24090dfc0 100644 --- a/llvm/tools/opt/opt.cpp +++ b/llvm/tools/opt/opt.cpp @@ -339,7 +339,8 @@ static bool shouldPinPassToLegacyPM(StringRef Pass) { "nvptx-", "mips-", "lanai-", "hexagon-", "bpf-", "avr-", "thumb2-", "arm-", "si-", "gcn-", "amdgpu-", "aarch64-", "amdgcn-", "polly-", "riscv-", "dxil-"}; - std::vector PassNameContain = {"ehprepare"}; + // TODO: remove "ehprepare" + std::vector PassNameContain = {"-eh-prepare", "ehprepare"}; std::vector PassNameExact = { "safe-stack", "cost-model", diff --git a/llvm/unittests/Analysis/ValueLatticeTest.cpp b/llvm/unittests/Analysis/ValueLatticeTest.cpp index ae221811f3fb5a84528d9687fc9628ac88640526..b456f286e0d96544b2ab590ae0f3b9b55dd01fcf 100644 --- a/llvm/unittests/Analysis/ValueLatticeTest.cpp +++ b/llvm/unittests/Analysis/ValueLatticeTest.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/ValueLattice.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/Constants.h" #include "llvm/IR/IRBuilder.h" diff --git a/llvm/unittests/Analysis/VectorFunctionABITest.cpp b/llvm/unittests/Analysis/VectorFunctionABITest.cpp index 81c1807cdcaa8135b66091095149e3bf2841ccfe..201dd1127ef2348cb794cb971a3623b8bd45a33f 100644 --- a/llvm/unittests/Analysis/VectorFunctionABITest.cpp +++ b/llvm/unittests/Analysis/VectorFunctionABITest.cpp @@ -1,4 +1,4 @@ -//===------- VectorFunctionABITest.cpp - VFABI Unittests ---------===// +//===------- VectorFunctionABITest.cpp - VFABI unit tests ---------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,6 +6,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/StringRef.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/AsmParser/Parser.h" #include "llvm/IR/InstIterator.h" @@ -14,24 +15,31 @@ using namespace llvm; namespace { -// Test fixture needed that holds the veariables needed by the parser. +/// Perform tests against VFABI Rules. `invokeParser` creates a VFInfo object +/// and a scalar FunctionType, which are used by tests to check that: +/// 1. The scalar and vector names are correct. +/// 2. The number of parameters from the parsed mangled name matches the number +/// of arguments in the scalar function passed as FunctionType string. +/// 3. The number of vector parameters and their types match the values +/// specified in the test. +/// On masked functions it also checks that the last parameter is a mask (ie, +/// GlobalPredicate). +/// 4. The vector function is correctly found to have a mask. +/// class VFABIParserTest : public ::testing::Test { private: // Parser output. VFInfo Info; - // Reset the data needed for the test. - void reset(const StringRef Name, const StringRef IRType) { + /// Reset the data needed for the test. + void reset(const StringRef ScalarFTyStr) { M = parseAssemblyString("declare void @dummy()", Err, Ctx); EXPECT_NE(M.get(), nullptr) << "Loading an invalid module.\n " << Err.getMessage() << "\n"; - Type *Ty = parseType(IRType, Err, *(M.get())); - FTy = dyn_cast(Ty); - EXPECT_NE(FTy, nullptr) << "Invalid function type string: " << IRType - << "\n" - << Err.getMessage() << "\n"; - F = M->getOrInsertFunction(Name, FTy); - EXPECT_NE(F.getCallee(), nullptr) - << "The function must be present in the module\n"; + Type *Ty = parseType(ScalarFTyStr, Err, *(M.get())); + ScalarFTy = dyn_cast(Ty); + EXPECT_NE(ScalarFTy, nullptr) + << "Invalid function type string: " << ScalarFTyStr << "\n" + << Err.getMessage() << "\n"; // Reset the VFInfo Info = VFInfo(); } @@ -40,69 +48,48 @@ private: LLVMContext Ctx; SMDiagnostic Err; std::unique_ptr M; - FunctionType *FTy; - FunctionCallee F; + FunctionType *ScalarFTy = nullptr; protected: - // Referencies to the parser output field. + // References to the parser output field. ElementCount &VF = Info.Shape.VF; VFISAKind &ISA = Info.ISA; + /// Parameters for the vectorized function SmallVector &Parameters = Info.Shape.Parameters; std::string &ScalarName = Info.ScalarName; std::string &VectorName = Info.VectorName; - // Invoke the parser. We need to make sure that a function exist in - // the module because the parser fails if such function don't - // exists. Every time this method is invoked the state of the test - // is reset. - // - // \p MangledName -> the string the parser has to demangle. - // - // \p VectorName -> optional vector name that the method needs to - // use to create the function in the module if it differs from the - // standard mangled name. - // - // \p IRType -> FunctionType string to be used for the signature of - // the vector function. The correct signature is needed by the - // parser only for scalable functions. For the sake of testing, the - // generic fixed-length case can use as signature `void()`. - // + + /// Invoke the parser. Every time this method is invoked the state of the test + /// is reset. + /// + /// \p MangledName string the parser has to demangle. + /// + /// \p ScalarFTyStr FunctionType string to get the signature of the scalar + /// function, which is used by `tryDemangleForVFABI` to check for the number + /// of arguments on scalable vectors, and by `matchParameters` to perform some + /// additional checking in the tests in this file. bool invokeParser(const StringRef MangledName, - const StringRef ScalarName = "", - const StringRef IRType = "void()") { - StringRef Name = MangledName; - if (!ScalarName.empty()) - Name = ScalarName; - // Reset the VFInfo and the Module to be able to invoke - // `invokeParser` multiple times in the same test. - reset(Name, IRType); - - // Fake the arguments to the CallInst. - SmallVector Args; - for (Type *ParamTy : FTy->params()) { - Args.push_back(Constant::getNullValue(ParamTy->getScalarType())); - } - std::unique_ptr CI(CallInst::Create(F, Args)); - const auto OptInfo = - VFABI::tryDemangleForVFABI(MangledName, CI->getFunctionType()); - if (OptInfo) { + const StringRef ScalarFTyStr = "void()") { + // Reset the VFInfo to be able to call `invokeParser` multiple times in + // the same test. + reset(ScalarFTyStr); + + const auto OptInfo = VFABI::tryDemangleForVFABI(MangledName, ScalarFTy); + if (OptInfo) Info = *OptInfo; - return true; - } - return false; + return OptInfo.has_value(); } - // Checks that 1. the last Parameter in the Shape is of type - // VFParamKind::GlobalPredicate and 2. it is the only one of such - // type. - bool IsMasked() const { - const auto NGlobalPreds = - std::count_if(Info.Shape.Parameters.begin(), - Info.Shape.Parameters.end(), [](const VFParameter PK) { - return PK.ParamKind == VFParamKind::GlobalPredicate; - }); - return NGlobalPreds == 1 && Info.Shape.Parameters.back().ParamKind == - VFParamKind::GlobalPredicate; + /// Returns whether the parsed function contains a mask. + bool isMasked() const { return Info.isMasked(); } + + /// Check if the number of vectorized parameters matches the scalar ones. This + /// requires a correct scalar FunctionType string to be fed to the + /// 'invokeParser'. Mask parameters that are only required by the vector + /// function call are ignored. + bool matchParametersNum() { + return (Parameters.size() - isMasked()) == ScalarFTy->getNumParams(); } }; } // unnamed namespace @@ -132,47 +119,74 @@ TEST_F(VFABIParserTest, OnlyValidNames) { EXPECT_FALSE(invokeParser("_ZGVnN2v_")); // Missing _ separator. EXPECT_FALSE(invokeParser("_ZGVnN2vfoo")); - // Missing . Using `fakename` because the string being - // parsed is not a valid function name that `invokeParser` can add. - EXPECT_FALSE(invokeParser("_ZGVnN2v_foo()", "fakename")); - // Unterminated name. Using `fakename` because the string being - // parsed is not a valid function name that `invokeParser` can add. - EXPECT_FALSE(invokeParser("_ZGVnN2v_foo(bar", "fakename")); + // Missing . + EXPECT_FALSE(invokeParser("_ZGVnN2v_foo()")); + // Unterminated name. + EXPECT_FALSE(invokeParser("_ZGVnN2v_foo(bar")); } TEST_F(VFABIParserTest, ParamListParsing) { - EXPECT_TRUE(invokeParser("_ZGVnN2vl16Ls32R3l_foo")); + EXPECT_TRUE( + invokeParser("_ZGVnN2vl16Ls32R3l_foo", "void(i32, i32, i32, ptr, i32)")); + EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_EQ(false, isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; EXPECT_EQ(Parameters.size(), (unsigned)5); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector, 0})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::OMP_Linear, 16})); EXPECT_EQ(Parameters[2], VFParameter({2, VFParamKind::OMP_LinearValPos, 32})); EXPECT_EQ(Parameters[3], VFParameter({3, VFParamKind::OMP_LinearRef, 3})); EXPECT_EQ(Parameters[4], VFParameter({4, VFParamKind::OMP_Linear, 1})); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "_ZGVnN2vl16Ls32R3l_foo"); } TEST_F(VFABIParserTest, ScalarNameAndVectorName_01) { - EXPECT_TRUE(invokeParser("_ZGVnM2v_sin")); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "_ZGVnM2v_sin"); + EXPECT_TRUE(invokeParser("_ZGVnM2v_foo(vector_foo)", "void(i32)")); + EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_EQ(true, isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ScalarNameAndVectorName_02) { - EXPECT_TRUE(invokeParser("_ZGVnM2v_sin(UserFunc)", "UserFunc")); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "UserFunc"); + EXPECT_TRUE(invokeParser("_ZGVnM2v_foo(vector_foo)", "void(i32)")); + EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_EQ(true, isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ScalarNameAndVectorName_03) { - EXPECT_TRUE(invokeParser("_ZGVnM2v___sin_sin_sin")); - EXPECT_EQ(ScalarName, "__sin_sin_sin"); - EXPECT_EQ(VectorName, "_ZGVnM2v___sin_sin_sin"); + EXPECT_TRUE( + invokeParser("_ZGVnM2v___foo_bar_abc(fooBarAbcVec)", "void(i32)")); + EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_EQ(true, isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(ScalarName, "__foo_bar_abc"); + EXPECT_EQ(VectorName, "fooBarAbcVec"); +} + +TEST_F(VFABIParserTest, ScalarNameOnly) { + EXPECT_TRUE(invokeParser("_ZGVnM2v___foo_bar_abc")); + EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_EQ(true, isMasked()); + EXPECT_EQ(ScalarName, "__foo_bar_abc"); + // no vector name specified (as it's optional), so it should have the entire + // mangled name. + EXPECT_EQ(VectorName, "_ZGVnM2v___foo_bar_abc"); } TEST_F(VFABIParserTest, Parse) { - EXPECT_TRUE(invokeParser("_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_FALSE(IsMasked()); + EXPECT_TRUE( + invokeParser("_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000_foo", + "void(i32, i32, i32, i32, ptr, i32, i32, i32, ptr)")); EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)9); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector, 0})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::OMP_LinearPos, 2})); @@ -183,75 +197,90 @@ TEST_F(VFABIParserTest, Parse) { EXPECT_EQ(Parameters[6], VFParameter({6, VFParamKind::OMP_LinearVal, 10})); EXPECT_EQ(Parameters[7], VFParameter({7, VFParamKind::OMP_LinearUVal, 100})); EXPECT_EQ(Parameters[8], VFParameter({8, VFParamKind::OMP_LinearRef, 1000})); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000_sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000_foo"); } TEST_F(VFABIParserTest, ParseVectorName) { - EXPECT_TRUE(invokeParser("_ZGVnN2v_sin(my_v_sin)", "my_v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_FALSE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVnN2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)1); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector, 0})); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "my_v_sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, LinearWithCompileTimeNegativeStep) { - EXPECT_TRUE(invokeParser("_ZGVnN2ln1Ln10Un100Rn1000_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_FALSE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVnN2ln1Ln10Un100Rn1000_foo(vector_foo)", + "void(i32, i32, i32, ptr)")); EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)4); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::OMP_Linear, -1})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::OMP_LinearVal, -10})); EXPECT_EQ(Parameters[2], VFParameter({2, VFParamKind::OMP_LinearUVal, -100})); EXPECT_EQ(Parameters[3], VFParameter({3, VFParamKind::OMP_LinearRef, -1000})); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "_ZGVnN2ln1Ln10Un100Rn1000_sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseScalableSVE) { - EXPECT_TRUE(invokeParser("_ZGVsMxv_sin(custom_vg)", "sin", "i32(i32)")); - EXPECT_EQ(VF, ElementCount::getScalable(4)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVsMxv_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::SVE); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "custom_vg"); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getScalable(4)); + EXPECT_EQ(Parameters.size(), (unsigned)2); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseFixedWidthSVE) { - EXPECT_TRUE(invokeParser("_ZGVsM2v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVsM2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::SVE); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "_ZGVsM2v_sin"); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); + EXPECT_EQ(Parameters.size(), (unsigned)2); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, NotAVectorFunctionABIName) { // Vector names should start with `_ZGV`. - EXPECT_FALSE(invokeParser("ZGVnN2v_sin")); + EXPECT_FALSE(invokeParser("ZGVnN2v_foo")); } TEST_F(VFABIParserTest, LinearWithRuntimeStep) { - EXPECT_FALSE(invokeParser("_ZGVnN2ls_sin")) + EXPECT_FALSE(invokeParser("_ZGVnN2ls_foo")) << "A number should be present after \"ls\"."; - EXPECT_TRUE(invokeParser("_ZGVnN2ls2_sin")); - EXPECT_FALSE(invokeParser("_ZGVnN2Rs_sin")) + EXPECT_TRUE(invokeParser("_ZGVnN2ls2_foo")); + EXPECT_FALSE(invokeParser("_ZGVnN2Rs_foo")) << "A number should be present after \"Rs\"."; - EXPECT_TRUE(invokeParser("_ZGVnN2Rs4_sin")); - EXPECT_FALSE(invokeParser("_ZGVnN2Ls_sin")) + EXPECT_TRUE(invokeParser("_ZGVnN2Rs4_foo")); + EXPECT_FALSE(invokeParser("_ZGVnN2Ls_foo")) << "A number should be present after \"Ls\"."; - EXPECT_TRUE(invokeParser("_ZGVnN2Ls6_sin")); - EXPECT_FALSE(invokeParser("_ZGVnN2Us_sin")) + EXPECT_TRUE(invokeParser("_ZGVnN2Ls6_foo")); + EXPECT_FALSE(invokeParser("_ZGVnN2Us_foo")) << "A number should be present after \"Us\"."; - EXPECT_TRUE(invokeParser("_ZGVnN2Us8_sin")); + EXPECT_TRUE(invokeParser("_ZGVnN2Us8_foo")); } TEST_F(VFABIParserTest, LinearWithoutCompileTime) { - EXPECT_TRUE(invokeParser("_ZGVnN3lLRUlnLnRnUn_sin")); + EXPECT_TRUE(invokeParser("_ZGVnN3lLRUlnLnRnUn_foo(vector_foo)", + "void(i32, i32, ptr, i32, i32, i32, ptr, i32)")); + EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; EXPECT_EQ(Parameters.size(), (unsigned)8); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::OMP_Linear, 1})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::OMP_LinearVal, 1})); @@ -261,80 +290,73 @@ TEST_F(VFABIParserTest, LinearWithoutCompileTime) { EXPECT_EQ(Parameters[5], VFParameter({5, VFParamKind::OMP_LinearVal, -1})); EXPECT_EQ(Parameters[6], VFParameter({6, VFParamKind::OMP_LinearRef, -1})); EXPECT_EQ(Parameters[7], VFParameter({7, VFParamKind::OMP_LinearUVal, -1})); -} - -TEST_F(VFABIParserTest, ISA) { - EXPECT_TRUE(invokeParser("_ZGVqN2v_sin")); - EXPECT_EQ(ISA, VFISAKind::Unknown); - - EXPECT_TRUE(invokeParser("_ZGVnN2v_sin")); - EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); - - EXPECT_TRUE(invokeParser("_ZGVsN2v_sin")); - EXPECT_EQ(ISA, VFISAKind::SVE); - - EXPECT_TRUE(invokeParser("_ZGVbN2v_sin")); - EXPECT_EQ(ISA, VFISAKind::SSE); - - EXPECT_TRUE(invokeParser("_ZGVcN2v_sin")); - EXPECT_EQ(ISA, VFISAKind::AVX); - - EXPECT_TRUE(invokeParser("_ZGVdN2v_sin")); - EXPECT_EQ(ISA, VFISAKind::AVX2); - - EXPECT_TRUE(invokeParser("_ZGVeN2v_sin")); - EXPECT_EQ(ISA, VFISAKind::AVX512); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, LLVM_ISA) { - EXPECT_FALSE(invokeParser("_ZGV_LLVM_N2v_sin")); - EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_sin_(vector_name)", "vector_name")); + EXPECT_FALSE(invokeParser("_ZGV_LLVM_N2v_foo")); + EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::LLVM); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(Parameters.size(), (unsigned)1); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, InvalidMask) { - EXPECT_FALSE(invokeParser("_ZGVsK2v_sin")); + EXPECT_FALSE(invokeParser("_ZGVsK2v_foo")); } TEST_F(VFABIParserTest, InvalidParameter) { - EXPECT_FALSE(invokeParser("_ZGVsM2vX_sin")); + EXPECT_FALSE(invokeParser("_ZGVsM2vX_foo")); } TEST_F(VFABIParserTest, Align) { - EXPECT_TRUE(invokeParser("_ZGVsN2l2a2_sin")); + EXPECT_TRUE(invokeParser("_ZGVsN2l2a2_foo(vector_foo)", "void(i32)")); + EXPECT_EQ(ISA, VFISAKind::SVE); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; EXPECT_EQ(Parameters.size(), (unsigned)1); EXPECT_EQ(Parameters[0].Alignment, Align(2)); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); // Missing alignment value. - EXPECT_FALSE(invokeParser("_ZGVsM2l2a_sin")); + EXPECT_FALSE(invokeParser("_ZGVsM2l2a_foo")); // Invalid alignment token "x". - EXPECT_FALSE(invokeParser("_ZGVsM2l2ax_sin")); + EXPECT_FALSE(invokeParser("_ZGVsM2l2ax_foo")); // Alignment MUST be associated to a paramater. - EXPECT_FALSE(invokeParser("_ZGVsM2a2_sin")); + EXPECT_FALSE(invokeParser("_ZGVsM2a2_foo")); // Alignment must be a power of 2. - EXPECT_FALSE(invokeParser("_ZGVsN2l2a0_sin")); - EXPECT_TRUE(invokeParser("_ZGVsN2l2a1_sin")); - EXPECT_FALSE(invokeParser("_ZGVsN2l2a3_sin")); - EXPECT_FALSE(invokeParser("_ZGVsN2l2a6_sin")); + EXPECT_FALSE(invokeParser("_ZGVsN2l2a0_foo")); + EXPECT_TRUE(invokeParser("_ZGVsN2l2a1_foo")); + EXPECT_FALSE(invokeParser("_ZGVsN2l2a3_foo")); + EXPECT_FALSE(invokeParser("_ZGVsN2l2a6_foo")); } TEST_F(VFABIParserTest, ParseUniform) { - EXPECT_TRUE(invokeParser("_ZGVnN2u_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_FALSE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVnN2u_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)1); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::OMP_Uniform, 0})); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "_ZGVnN2u_sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); // Uniform doesn't expect extra data. - EXPECT_FALSE(invokeParser("_ZGVnN2u0_sin")); + EXPECT_FALSE(invokeParser("_ZGVnN2u0_foo")); } TEST_F(VFABIParserTest, ISAIndependentMangling) { // This test makes sure that the mangling of the parameters in // independent on the token. + const StringRef IRTy = + "void(i32, i32, i32, i32, ptr, i32, i32, i32, i32, i32)"; const SmallVector ExpectedParams = { VFParameter({0, VFParamKind::Vector, 0}), VFParameter({1, VFParamKind::OMP_LinearPos, 2}), @@ -351,61 +373,63 @@ TEST_F(VFABIParserTest, ISAIndependentMangling) { #define __COMMON_CHECKS \ do { \ EXPECT_EQ(VF, ElementCount::getFixed(2)); \ - EXPECT_FALSE(IsMasked()); \ + EXPECT_FALSE(isMasked()); \ + EXPECT_TRUE(matchParametersNum()) \ + << "Different number of scalar parameters"; \ EXPECT_EQ(Parameters.size(), (unsigned)10); \ EXPECT_EQ(Parameters, ExpectedParams); \ - EXPECT_EQ(ScalarName, "sin"); \ + EXPECT_EQ(ScalarName, "foo"); \ + EXPECT_EQ(VectorName, "vector_foo"); \ } while (0) // Advanced SIMD: = "n" - EXPECT_TRUE(invokeParser("_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000u_sin")); + EXPECT_TRUE(invokeParser( + "_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000u_sin"); // SVE: = "s" - EXPECT_TRUE(invokeParser("_ZGVsN2vls2Ls27Us4Rs5l1L10U100R1000u_sin")); + EXPECT_TRUE(invokeParser( + "_ZGVsN2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::SVE); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "_ZGVsN2vls2Ls27Us4Rs5l1L10U100R1000u_sin"); // SSE: = "b" - EXPECT_TRUE(invokeParser("_ZGVbN2vls2Ls27Us4Rs5l1L10U100R1000u_sin")); + EXPECT_TRUE(invokeParser( + "_ZGVbN2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::SSE); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "_ZGVbN2vls2Ls27Us4Rs5l1L10U100R1000u_sin"); // AVX: = "c" - EXPECT_TRUE(invokeParser("_ZGVcN2vls2Ls27Us4Rs5l1L10U100R1000u_sin")); + EXPECT_TRUE(invokeParser( + "_ZGVcN2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::AVX); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "_ZGVcN2vls2Ls27Us4Rs5l1L10U100R1000u_sin"); // AVX2: = "d" - EXPECT_TRUE(invokeParser("_ZGVdN2vls2Ls27Us4Rs5l1L10U100R1000u_sin")); + EXPECT_TRUE(invokeParser( + "_ZGVdN2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::AVX2); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "_ZGVdN2vls2Ls27Us4Rs5l1L10U100R1000u_sin"); // AVX512: = "e" - EXPECT_TRUE(invokeParser("_ZGVeN2vls2Ls27Us4Rs5l1L10U100R1000u_sin")); + EXPECT_TRUE(invokeParser( + "_ZGVeN2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::AVX512); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "_ZGVeN2vls2Ls27Us4Rs5l1L10U100R1000u_sin"); // LLVM: = "_LLVM_" internal vector function. EXPECT_TRUE(invokeParser( - "_ZGV_LLVM_N2vls2Ls27Us4Rs5l1L10U100R1000u_sin(vectorf)", "vectorf")); + "_ZGV_LLVM_N2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::LLVM); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "vectorf"); // Unknown ISA (randomly using "q"). This test will need update if // some targets decide to use "q" as their ISA token. - EXPECT_TRUE(invokeParser("_ZGVqN2vls2Ls27Us4Rs5l1L10U100R1000u_sin")); + EXPECT_TRUE(invokeParser( + "_ZGVqN2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::Unknown); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "_ZGVqN2vls2Ls27Us4Rs5l1L10U100R1000u_sin"); #undef __COMMON_CHECKS } @@ -423,108 +447,167 @@ TEST_F(VFABIParserTest, MissingVectorNameTermination) { } TEST_F(VFABIParserTest, ParseMaskingNEON) { - EXPECT_TRUE(invokeParser("_ZGVnM2v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVnM2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); - EXPECT_EQ(ScalarName, "sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseMaskingSVE) { - EXPECT_TRUE(invokeParser("_ZGVsM2v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVsM2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::SVE); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); - EXPECT_EQ(ScalarName, "sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseMaskingSSE) { - EXPECT_TRUE(invokeParser("_ZGVbM2v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVbM2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::SSE); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); - EXPECT_EQ(ScalarName, "sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseMaskingAVX) { - EXPECT_TRUE(invokeParser("_ZGVcM2v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVcM2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::AVX); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); - EXPECT_EQ(ScalarName, "sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseMaskingAVX2) { - EXPECT_TRUE(invokeParser("_ZGVdM2v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVdM2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::AVX2); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); - EXPECT_EQ(ScalarName, "sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseMaskingAVX512) { - EXPECT_TRUE(invokeParser("_ZGVeM2v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVeM2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::AVX512); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); - EXPECT_EQ(ScalarName, "sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseMaskingLLVM) { - EXPECT_TRUE(invokeParser("_ZGV_LLVM_M2v_sin(custom_vector_sin)", - "custom_vector_sin")); + EXPECT_TRUE(invokeParser("_ZGV_LLVM_M2v_foo(vector_foo)", "void(i32)")); + EXPECT_EQ(ISA, VFISAKind::LLVM); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_EQ(Parameters.size(), (unsigned)2); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); +} + +TEST_F(VFABIParserTest, ParseScalableMaskingLLVM) { + EXPECT_FALSE(invokeParser("_ZGV_LLVM_Mxv_foo(vector_foo)")); +} + +TEST_F(VFABIParserTest, LLVM_InternalISA) { + EXPECT_FALSE(invokeParser("_ZGV_LLVM_N2v_foo")); + EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::LLVM); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(Parameters.size(), (unsigned)1); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); +} + +TEST_F(VFABIParserTest, IntrinsicsInLLVMIsa) { + EXPECT_TRUE(invokeParser("_ZGV_LLVM_N4vv_llvm.pow.f32(__svml_powf4)", + "void(float, float)")); + EXPECT_EQ(ISA, VFISAKind::LLVM); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(4)); + EXPECT_EQ(Parameters.size(), (unsigned)2); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::Vector})); + EXPECT_EQ(ScalarName, "llvm.pow.f32"); + EXPECT_EQ(VectorName, "__svml_powf4"); +} + +TEST_F(VFABIParserTest, ParseScalableRequiresDeclaration) { + const char *MangledName = "_ZGVsMxv_sin(custom_vg)"; + EXPECT_FALSE(invokeParser(MangledName)); + EXPECT_TRUE(invokeParser(MangledName, "void(i32)")); + EXPECT_EQ(ISA, VFISAKind::SVE); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "custom_vector_sin"); + EXPECT_EQ(VectorName, "custom_vg"); } -TEST_F(VFABIParserTest, ParseScalableMaskingLLVM) { - EXPECT_FALSE( - invokeParser("_ZGV_LLVM_Mxv_sin(custom_vector_sin)", "sin", "i32(i32)")); +TEST_F(VFABIParserTest, ZeroIsInvalidVLEN) { + EXPECT_FALSE(invokeParser("_ZGVeM0v_foo")); + EXPECT_FALSE(invokeParser("_ZGVeN0v_foo")); + EXPECT_FALSE(invokeParser("_ZGVsM0v_foo")); + EXPECT_FALSE(invokeParser("_ZGVsN0v_foo")); } TEST_F(VFABIParserTest, ParseScalableMaskingSVE) { - EXPECT_TRUE( - invokeParser("_ZGVsMxv_sin(custom_vector_sin)", "sin", "i32(i32)")); - EXPECT_TRUE(IsMasked()); - EXPECT_EQ(VF, ElementCount::getScalable(4)); + EXPECT_TRUE(invokeParser("_ZGVsMxv_foo(vector_foo)", "i32(i32)")); EXPECT_EQ(ISA, VFISAKind::SVE); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getScalable(4)); EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "custom_vector_sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseScalableMaskingSVESincos) { EXPECT_TRUE(invokeParser("_ZGVsMxvl8l8_sincos(custom_vector_sincos)", - "sincos", "void(double, double *, double *)")); - EXPECT_EQ(VF, ElementCount::getScalable(2)); - EXPECT_TRUE(IsMasked()); + "void(double, ptr, ptr)")); EXPECT_EQ(ISA, VFISAKind::SVE); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getScalable(2)); EXPECT_EQ(Parameters.size(), (unsigned)4); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::OMP_Linear, 8})); @@ -537,27 +620,42 @@ TEST_F(VFABIParserTest, ParseScalableMaskingSVESincos) { // Make sure that we get the correct VF if the return type is wider than any // parameter type. TEST_F(VFABIParserTest, ParseWiderReturnTypeSVE) { - EXPECT_TRUE( - invokeParser("_ZGVsMxvv_foo(vector_foo)", "foo", "i64(i32, i32)")); + EXPECT_TRUE(invokeParser("_ZGVsMxvv_foo(vector_foo)", "i64(i32, i32)")); + EXPECT_EQ(ISA, VFISAKind::SVE); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(Parameters.size(), (unsigned)3); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::Vector})); + EXPECT_EQ(Parameters[2], VFParameter({2, VFParamKind::GlobalPredicate})); EXPECT_EQ(VF, ElementCount::getScalable(2)); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } // Make sure we handle void return types. TEST_F(VFABIParserTest, ParseVoidReturnTypeSVE) { - EXPECT_TRUE(invokeParser("_ZGVsMxv_foo(vector_foo)", "foo", "void(i16)")); + EXPECT_TRUE(invokeParser("_ZGVsMxv_foo(vector_foo)", "void(i16)")); + EXPECT_EQ(ISA, VFISAKind::SVE); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(Parameters.size(), (unsigned)2); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); EXPECT_EQ(VF, ElementCount::getScalable(8)); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } // Make sure we reject unsupported parameter types. TEST_F(VFABIParserTest, ParseUnsupportedElementTypeSVE) { - EXPECT_FALSE(invokeParser("_ZGVsMxv_foo(vector_foo)", "foo", "void(i128)")); + EXPECT_FALSE(invokeParser("_ZGVsMxv_foo(vector_foo)", "void(i128)")); } // Make sure we reject unsupported return types TEST_F(VFABIParserTest, ParseUnsupportedReturnTypeSVE) { - EXPECT_FALSE(invokeParser("_ZGVsMxv_foo(vector_foo)", "foo", "fp128(float)")); + EXPECT_FALSE(invokeParser("_ZGVsMxv_foo(vector_foo)", "fp128(float)")); } - class VFABIAttrTest : public testing::Test { protected: void SetUp() override { @@ -592,39 +690,6 @@ TEST_F(VFABIAttrTest, Read) { EXPECT_EQ(Mappings, Exp); } -TEST_F(VFABIParserTest, LLVM_InternalISA) { - EXPECT_FALSE(invokeParser("_ZGV_LLVM_N2v_sin")); - EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_sin_(vector_name)", "vector_name")); - EXPECT_EQ(ISA, VFISAKind::LLVM); -} - -TEST_F(VFABIParserTest, IntrinsicsInLLVMIsa) { - EXPECT_TRUE(invokeParser("_ZGV_LLVM_N4vv_llvm.pow.f32(__svml_powf4)", - "__svml_powf4")); - EXPECT_EQ(VF, ElementCount::getFixed(4)); - EXPECT_FALSE(IsMasked()); - EXPECT_EQ(ISA, VFISAKind::LLVM); - EXPECT_EQ(Parameters.size(), (unsigned)2); - EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); - EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::Vector})); - EXPECT_EQ(ScalarName, "llvm.pow.f32"); -} - -TEST_F(VFABIParserTest, ParseScalableRequiresDeclaration) { - const char *MangledName = "_ZGVsMxv_sin(custom_vg)"; - // The parser succeds only when the correct function definition of - // `custom_vg` is added to the module. - EXPECT_FALSE(invokeParser(MangledName)); - EXPECT_TRUE(invokeParser(MangledName, "sin", "double(double)")); -} - -TEST_F(VFABIParserTest, ZeroIsInvalidVLEN) { - EXPECT_FALSE(invokeParser("_ZGVeM0v_sin")); - EXPECT_FALSE(invokeParser("_ZGVeN0v_sin")); - EXPECT_FALSE(invokeParser("_ZGVsM0v_sin")); - EXPECT_FALSE(invokeParser("_ZGVsN0v_sin")); -} - static std::unique_ptr parseIR(LLVMContext &C, const char *IR) { SMDiagnostic Err; std::unique_ptr Mod = parseAssemblyString(IR, Err, C); @@ -642,9 +707,9 @@ entry: ret void } )IR"); - auto F = dyn_cast_or_null(M->getNamedValue("call")); + auto *F = dyn_cast_or_null(M->getNamedValue("call")); ASSERT_TRUE(F); - auto CI = dyn_cast(&F->front().front()); + auto *CI = dyn_cast(&F->front().front()); ASSERT_TRUE(CI); ASSERT_TRUE(CI->isIndirectCall()); auto Mappings = VFDatabase::getMappings(*CI); diff --git a/llvm/unittests/Analysis/VectorUtilsTest.cpp b/llvm/unittests/Analysis/VectorUtilsTest.cpp index 1b3a8b0259f0176cee296a196b07821dd93b75f8..14958aa646a04dc3f6a9dfb522deb103ae99d4da 100644 --- a/llvm/unittests/Analysis/VectorUtilsTest.cpp +++ b/llvm/unittests/Analysis/VectorUtilsTest.cpp @@ -17,7 +17,6 @@ #include "llvm/IR/NoFolder.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/SourceMgr.h" -#include "llvm/Support/KnownBits.h" #include "gtest/gtest.h" using namespace llvm; diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp index 0b7f8f41bc53f435a6a4565041f525930fa38b8c..5cb0310c0ad097762177df7e12c35e788576d2e8 100644 --- a/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp +++ b/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp @@ -1615,6 +1615,42 @@ TEST(DWARFDebugInfo, TestFindRecurse) { EXPECT_EQ(AbsDieName, StringOpt.value_or(nullptr)); } +TEST(DWARFDebugInfo, TestSelfRecursiveType) { + typedef uint32_t AddrType; + Triple Triple = getDefaultTargetTripleForAddrSize(sizeof(AddrType)); + if (!isConfigurationSupported(Triple)) + GTEST_SKIP(); + + auto ExpectedDG = dwarfgen::Generator::create(Triple, 4); + ASSERT_THAT_EXPECTED(ExpectedDG, Succeeded()); + dwarfgen::Generator *DG = ExpectedDG.get().get(); + dwarfgen::CompileUnit &CU = DG->addCompileUnit(); + dwarfgen::DIE CUDie = CU.getUnitDIE(); + + // Create an invalid self-recursive typedef. + dwarfgen::DIE TypedefDie = CUDie.addChild(DW_TAG_typedef); + TypedefDie.addAttribute(DW_AT_name, DW_FORM_strp, "illegal"); + TypedefDie.addAttribute(DW_AT_type, DW_FORM_ref_addr, TypedefDie); + + MemoryBufferRef FileBuffer(DG->generate(), "dwarf"); + auto Obj = object::ObjectFile::createObjectFile(FileBuffer); + EXPECT_TRUE((bool)Obj); + std::unique_ptr DwarfContext = DWARFContext::create(**Obj); + + // Verify the number of compile units is correct. + uint32_t NumCUs = DwarfContext->getNumCompileUnits(); + EXPECT_EQ(NumCUs, 1u); + DWARFCompileUnit *U = cast(DwarfContext->getUnitAtIndex(0)); + { + DWARFDie CUDie = U->getUnitDIE(false); + EXPECT_TRUE(CUDie.isValid()); + DWARFDie TypedefDie = CUDie.getFirstChild(); + + // Test that getTypeSize doesn't get into an infinite loop. + EXPECT_EQ(TypedefDie.getTypeSize(sizeof(AddrType)), std::nullopt); + } +} + TEST(DWARFDebugInfo, TestDwarfToFunctions) { // Test all of the dwarf::toXXX functions that take a // std::optional and extract the values from it. diff --git a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt index 37768e91fd44729036402c5a068f153e04ce95ad..f102ba59e3754256e4f72b2264b5e9ff274ce3ef 100644 --- a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt @@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS IRReader JITLink Object + OrcDebugging OrcJIT OrcShared OrcTargetProcess diff --git a/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp b/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp index cbdd4af47e1d47da5d4d53147676111c3b98efe2..65d2f57234d0a59ed0566694212f00f8574dfc8e 100644 --- a/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp @@ -9,10 +9,12 @@ #include "llvm-c/Core.h" #include "llvm-c/Error.h" #include "llvm-c/LLJIT.h" +#include "llvm-c/LLJITUtils.h" #include "llvm-c/Orc.h" #include "gtest/gtest.h" #include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IRReader/IRReader.h" @@ -211,6 +213,20 @@ constexpr StringRef SumExample = } )"; +constexpr StringRef SumDebugExample = + R"( + define i32 @sum(i32 %x, i32 %y) { + entry: + %r = add nsw i32 %x, %y + ret i32 %r + } + !llvm.module.flags = !{!0} + !llvm.dbg.cu = !{!1} + !0 = !{i32 2, !"Debug Info Version", i32 3} + !1 = distinct !DICompileUnit(language: DW_LANG_C99, file: !2, emissionKind: FullDebug) + !2 = !DIFile(filename: "sum.c", directory: "/tmp") + )"; + } // end anonymous namespace. // Consumes the given error ref and returns the string error message. @@ -494,6 +510,75 @@ TEST_F(OrcCAPITestBase, AddObjectBuffer) { ASSERT_TRUE(!!SumAddr); } +// This must be kept in sync with gdb/gdb/jit.h . +extern "C" { + +typedef enum { + JIT_NOACTION = 0, + JIT_REGISTER_FN, + JIT_UNREGISTER_FN +} jit_actions_t; + +struct jit_code_entry { + struct jit_code_entry *next_entry; + struct jit_code_entry *prev_entry; + const char *symfile_addr; + uint64_t symfile_size; +}; + +struct jit_descriptor { + uint32_t version; + // This should be jit_actions_t, but we want to be specific about the + // bit-width. + uint32_t action_flag; + struct jit_code_entry *relevant_entry; + struct jit_code_entry *first_entry; +}; + +// We put information about the JITed function in this global, which the +// debugger reads. Make sure to specify the version statically, because the +// debugger checks the version before we can set it during runtime. +extern struct jit_descriptor __jit_debug_descriptor; + +static void *findLastDebugDescriptorEntryPtr() { + struct jit_code_entry *Last = __jit_debug_descriptor.first_entry; + while (Last && Last->next_entry) + Last = Last->next_entry; + return Last; +} +} + +#if defined(_AIX) or not(defined(__ELF__) or defined(__MACH__)) +TEST_F(OrcCAPITestBase, DISABLED_EnableDebugSupport) { +#else +static LLVM_ATTRIBUTE_USED void linkComponents() { + errs() << "Linking in runtime functions\n" + << (void *)&llvm_orc_registerJITLoaderGDBWrapper << '\n' + << (void *)&llvm_orc_registerJITLoaderGDBAllocAction << '\n'; +} +TEST_F(OrcCAPITestBase, EnableDebugSupport) { +#endif + if (LLVMErrorRef E = LLVMOrcLLJITEnableDebugSupport(Jit)) + FAIL() << "Error testing LLJIT debug support (triple = " << TargetTriple + << "): " << toString(E); + + void *Before = findLastDebugDescriptorEntryPtr(); + LLVMMemoryBufferRef ObjBuffer = createTestObject(SumDebugExample, "sum.ll"); + LLVMOrcObjectLayerRef ObjLayer = LLVMOrcLLJITGetObjLinkingLayer(Jit); + if (LLVMErrorRef E = + LLVMOrcObjectLayerAddObjectFile(ObjLayer, MainDylib, ObjBuffer)) + FAIL() << "Failed to add object file to ObjLinkingLayer (triple = " + << TargetTriple << "): " << toString(E); + + LLVMOrcJITTargetAddress SumAddr; + if (LLVMErrorRef E = LLVMOrcLLJITLookup(Jit, &SumAddr, "sum")) + FAIL() << "Symbol \"sum\" was not added into JIT (triple = " << TargetTriple + << "): " << toString(E); + + void *After = findLastDebugDescriptorEntryPtr(); + ASSERT_NE(Before, After); +} + #if defined(_AIX) TEST_F(OrcCAPITestBase, DISABLED_ExecutionTest) { #else diff --git a/llvm/unittests/Support/Chrono.cpp b/llvm/unittests/Support/Chrono.cpp index daf8a8a350f08c748ba05af44f43f3faca8e2c61..7dfc5dd2c29348ea8df9ce87c80f357aaad1a73b 100644 --- a/llvm/unittests/Support/Chrono.cpp +++ b/llvm/unittests/Support/Chrono.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "llvm/Support/Chrono.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/Support/FormatVariadic.h" #include "gtest/gtest.h" diff --git a/llvm/unittests/Support/CommandLineTest.cpp b/llvm/unittests/Support/CommandLineTest.cpp index 381fe70b6b4815649b7ad6a8cf4556f577493032..762ac0ea9c36dde9340fd0da4a99199ba44ff876 100644 --- a/llvm/unittests/Support/CommandLineTest.cpp +++ b/llvm/unittests/Support/CommandLineTest.cpp @@ -28,7 +28,6 @@ #include #include #include -#include using namespace llvm; using llvm::unittest::TempDir; @@ -1347,29 +1346,32 @@ struct AutoDeleteFile { } }; +static std::string interceptStdout(std::function F) { + outs().flush(); // flush any output from previous tests + AutoDeleteFile File; + { + OutputRedirector Stdout(fileno(stdout)); + if (!Stdout.Valid) + return ""; + File.FilePath = Stdout.FilePath; + F(); + outs().flush(); + } + auto Buffer = MemoryBuffer::getFile(File.FilePath); + if (!Buffer) + return ""; + return Buffer->get()->getBuffer().str(); +} + template class PrintOptionTestBase : public ::testing::Test { public: // Return std::string because the output of a failing EXPECT check is // unreadable for StringRef. It also avoids any lifetime issues. template std::string runTest(Ts... OptionAttributes) { - outs().flush(); // flush any output from previous tests - AutoDeleteFile File; - { - OutputRedirector Stdout(fileno(stdout)); - if (!Stdout.Valid) - return ""; - File.FilePath = Stdout.FilePath; - - StackOption TestOption(Opt, cl::desc(HelpText), - OptionAttributes...); - Func(TestOption); - outs().flush(); - } - auto Buffer = MemoryBuffer::getFile(File.FilePath); - if (!Buffer) - return ""; - return Buffer->get()->getBuffer().str(); + StackOption TestOption(Opt, cl::desc(HelpText), + OptionAttributes...); + return interceptStdout([&]() { Func(TestOption); }); } enum class OptionValue { Val }; @@ -2206,4 +2208,40 @@ TEST(CommandLineTest, DefaultValue) { EXPECT_EQ(1, StrInitOption.getNumOccurrences()); } +TEST(CommandLineTest, HelpWithoutSubcommands) { + // Check that the help message does not contain the "[subcommand]" placeholder + // and the "SUBCOMMANDS" section if there are no subcommands. + cl::ResetCommandLineParser(); + StackOption Opt("opt", cl::init(false)); + const char *args[] = {"prog"}; + EXPECT_TRUE(cl::ParseCommandLineOptions(std::size(args), args, StringRef(), + &llvm::nulls())); + auto Output = interceptStdout([]() { cl::PrintHelpMessage(); }); + EXPECT_NE(std::string::npos, Output.find("USAGE: prog [options]")) << Output; + EXPECT_EQ(std::string::npos, Output.find("SUBCOMMANDS:")) << Output; + cl::ResetCommandLineParser(); +} + +TEST(CommandLineTest, HelpWithSubcommands) { + // Check that the help message contains the "[subcommand]" placeholder in the + // "USAGE" line and describes subcommands. + cl::ResetCommandLineParser(); + StackSubCommand SC1("sc1", "First Subcommand"); + StackSubCommand SC2("sc2", "Second Subcommand"); + StackOption SC1Opt("sc1", cl::sub(SC1), cl::init(false)); + StackOption SC2Opt("sc2", cl::sub(SC2), cl::init(false)); + const char *args[] = {"prog"}; + EXPECT_TRUE(cl::ParseCommandLineOptions(std::size(args), args, StringRef(), + &llvm::nulls())); + auto Output = interceptStdout([]() { cl::PrintHelpMessage(); }); + EXPECT_NE(std::string::npos, + Output.find("USAGE: prog [subcommand] [options]")) + << Output; + EXPECT_NE(std::string::npos, Output.find("SUBCOMMANDS:")) << Output; + EXPECT_NE(std::string::npos, Output.find("sc1 - First Subcommand")) << Output; + EXPECT_NE(std::string::npos, Output.find("sc2 - Second Subcommand")) + << Output; + cl::ResetCommandLineParser(); +} + } // anonymous namespace diff --git a/llvm/unittests/Support/InstructionCostTest.cpp b/llvm/unittests/Support/InstructionCostTest.cpp index 2f634110bd513beefb580470610d455cec72ab18..4e2842d8ada9744a3bf046c60b236e5a76456b1d 100644 --- a/llvm/unittests/Support/InstructionCostTest.cpp +++ b/llvm/unittests/Support/InstructionCostTest.cpp @@ -8,7 +8,6 @@ #include "llvm/Support/InstructionCost.h" #include "gtest/gtest.h" -#include using namespace llvm; diff --git a/llvm/unittests/Support/MemoryTest.cpp b/llvm/unittests/Support/MemoryTest.cpp index 019ece9c172597b0521c4dd8c38b85380d1dd715..0164492829ccd7a93d00935477bc615408825876 100644 --- a/llvm/unittests/Support/MemoryTest.cpp +++ b/llvm/unittests/Support/MemoryTest.cpp @@ -9,7 +9,6 @@ #include "llvm/Support/Memory.h" #include "llvm/Support/Process.h" #include "gtest/gtest.h" -#include #include #if defined(__NetBSD__) diff --git a/llvm/unittests/Support/Path.cpp b/llvm/unittests/Support/Path.cpp index 35a01aa276679336118f60e2cce85f89c7b2bc26..a7b7e6a0f5044dd776b3ed39833885ad4a76dba5 100644 --- a/llvm/unittests/Support/Path.cpp +++ b/llvm/unittests/Support/Path.cpp @@ -882,7 +882,7 @@ TEST_F(FileSystemTest, TempFiles) { int FD2; SmallString<64> TempPath2; ASSERT_NO_ERROR(fs::createTemporaryFile("prefix", "temp", FD2, TempPath2)); - ASSERT_TRUE(TempPath2.endswith(".temp")); + ASSERT_TRUE(TempPath2.ends_with(".temp")); ASSERT_NE(TempPath.str(), TempPath2.str()); fs::file_status A, B; @@ -908,7 +908,7 @@ TEST_F(FileSystemTest, TempFiles) { SmallString<64> TempPath3; ASSERT_NO_ERROR(fs::createTemporaryFile("prefix", "", TempPath3)); - ASSERT_FALSE(TempPath3.endswith(".")); + ASSERT_FALSE(TempPath3.ends_with(".")); FileRemover Cleanup3(TempPath3); // Create a hard link to Temp1. @@ -1515,13 +1515,13 @@ TEST(Support, NormalizePath) { const char *Path7a = "~/aaa"; SmallString<64> Path7(Path7a); path::native(Path7, path::Style::windows_backslash); - EXPECT_TRUE(Path7.endswith("\\aaa")); - EXPECT_TRUE(Path7.startswith(PathHome)); + EXPECT_TRUE(Path7.ends_with("\\aaa")); + EXPECT_TRUE(Path7.starts_with(PathHome)); EXPECT_EQ(Path7.size(), PathHome.size() + strlen(Path7a + 1)); Path7 = Path7a; path::native(Path7, path::Style::windows_slash); - EXPECT_TRUE(Path7.endswith("/aaa")); - EXPECT_TRUE(Path7.startswith(PathHome)); + EXPECT_TRUE(Path7.ends_with("/aaa")); + EXPECT_TRUE(Path7.starts_with(PathHome)); EXPECT_EQ(Path7.size(), PathHome.size() + strlen(Path7a + 1)); const char *Path8a = "~"; diff --git a/llvm/unittests/TableGen/AutomataTest.cpp b/llvm/unittests/TableGen/AutomataTest.cpp index 53f66127292f8f2af6d13212151e4f09595c488c..833cf51b19a1008b15c940618e45e96554ed4dd4 100644 --- a/llvm/unittests/TableGen/AutomataTest.cpp +++ b/llvm/unittests/TableGen/AutomataTest.cpp @@ -6,7 +6,6 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Automaton.h" diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp index abd87b455acebf675edb4a8bdbc67adc8a42f9ac..687b56cc5aa20bcd94c9c3ffb3c4bfa7a1b072fb 100644 --- a/llvm/unittests/TargetParser/TargetParserTest.cpp +++ b/llvm/unittests/TargetParser/TargetParserTest.cpp @@ -416,13 +416,13 @@ INSTANTIATE_TEST_SUITE_P( ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_FP16 | ARM::AEK_RAS | ARM::AEK_DOTPROD, "8.2-A"), - ARMCPUTestParams("neoverse-n2", "armv8.5-a", "crypto-neon-fp-armv8", + ARMCPUTestParams("neoverse-n2", "armv9-a", "neon-fp-armv8", ARM::AEK_CRC | ARM::AEK_HWDIVTHUMB | ARM::AEK_HWDIVARM | ARM::AEK_MP | ARM::AEK_SEC | ARM::AEK_VIRT | ARM::AEK_DSP | ARM::AEK_BF16 | ARM::AEK_DOTPROD | ARM::AEK_RAS | ARM::AEK_I8MM | ARM::AEK_SB, - "8.5-A"), + "9-A"), ARMCPUTestParams("neoverse-v1", "armv8.4-a", "crypto-neon-fp-armv8", ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | @@ -469,13 +469,19 @@ INSTANTIATE_TEST_SUITE_P( ARM::AEK_FP | ARM::AEK_RAS | ARM::AEK_LOB | ARM::AEK_FP16 | ARM::AEK_PACBTI, "8.1-M.Mainline"), + ARMCPUTestParams("cortex-m52", "armv8.1-m.main", + "fp-armv8-fullfp16-d16", + ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_SIMD | + ARM::AEK_FP | ARM::AEK_RAS | ARM::AEK_LOB | + ARM::AEK_FP16 | ARM::AEK_PACBTI, + "8.1-M.Mainline"), ARMCPUTestParams("iwmmxt", "iwmmxt", "none", ARM::AEK_NONE, "iwmmxt"), ARMCPUTestParams("xscale", "xscale", "none", ARM::AEK_NONE, "xscale"), ARMCPUTestParams("swift", "armv7s", "neon-vfpv4", ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP, "7-S"))); -static constexpr unsigned NumARMCPUArchs = 89; +static constexpr unsigned NumARMCPUArchs = 90; TEST(TargetParserTest, testARMCPUArchList) { SmallVector List; @@ -1515,11 +1521,9 @@ INSTANTIATE_TEST_SUITE_P( AArch64::AEK_SSBS})), "8.2-A"), ARMCPUTestParams( - "neoverse-n2", "armv8.5-a", "crypto-neon-fp-armv8", + "neoverse-n2", "armv9-a", "crypto-neon-fp-armv8", (AArch64::ExtensionBitset( - {AArch64::AEK_CRC, AArch64::AEK_AES, - AArch64::AEK_SHA2, AArch64::AEK_SHA3, - AArch64::AEK_SM4, AArch64::AEK_FP, + {AArch64::AEK_CRC, AArch64::AEK_FP, AArch64::AEK_SIMD, AArch64::AEK_FP16, AArch64::AEK_RAS, AArch64::AEK_LSE, AArch64::AEK_SVE, AArch64::AEK_DOTPROD, diff --git a/llvm/unittests/TextAPI/RecordTests.cpp b/llvm/unittests/TextAPI/RecordTests.cpp index 076137de6ff1f4f0cda78eb9ace7064b3b5abf06..dbe58ca0eea59d7810c2d1e0e107aef8624d0c06 100644 --- a/llvm/unittests/TextAPI/RecordTests.cpp +++ b/llvm/unittests/TextAPI/RecordTests.cpp @@ -48,7 +48,7 @@ TEST(TAPIRecord, SimpleObjC) { TEST(TAPIRecord, SimpleSlice) { Triple T("arm64-apple-macosx13.3"); RecordsSlice Slice(T); - EXPECT_TRUE(Slice.isEmpty()); + EXPECT_TRUE(Slice.empty()); Slice.addRecord("_OBJC_CLASS_$_NSObject", SymbolFlags::None, GlobalRecord::Kind::Unknown, RecordLinkage::Rexported); Slice.addRecord("_OBJC_METACLASS_$_NSObject", SymbolFlags::None, @@ -59,7 +59,7 @@ TEST(TAPIRecord, SimpleSlice) { GlobalRecord::Kind::Unknown, RecordLinkage::Exported); Slice.addRecord("_foo", SymbolFlags::WeakDefined | SymbolFlags::Rexported, GlobalRecord::Kind::Variable, RecordLinkage::Rexported); - EXPECT_FALSE(Slice.isEmpty()); + EXPECT_FALSE(Slice.empty()); // Check global. EXPECT_FALSE(Slice.findGlobal("_foo", GlobalRecord::Kind::Function)); @@ -108,7 +108,7 @@ TEST(TAPIRecord, SimpleSlice) { TEST(TAPIRecord, LibraryAttrs) { Triple T("arm64-apple-ios15.1"); RecordsSlice Slice(T); - EXPECT_TRUE(Slice.isEmpty()); + EXPECT_TRUE(Slice.empty()); auto BA = Slice.getBinaryAttrs(); EXPECT_TRUE(Slice.hasBinaryAttrs()); diff --git a/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp index 2bab3aa15baadcb9bdec58f283075c2484f021be..1638ede7f3d4290b2383f76f31d62d254d572261 100644 --- a/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp +++ b/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp @@ -209,6 +209,16 @@ TEST_F(X86SnippetFileTest, MemoryMappingNoDefinition) { consumeError(std::move(Error)); } +TEST_F(X86SnippetFileTest, SnippetAddress) { + auto Snippets = TestCommon(R"( + # LLVM-EXEGESIS-SNIPPET-ADDRESS 10000 + )"); + ASSERT_TRUE(static_cast(Snippets)); + EXPECT_THAT(*Snippets, SizeIs(1)); + const auto &Snippet = (*Snippets)[0]; + EXPECT_EQ(Snippet.Key.SnippetAddress, 0x10000); +} + } // namespace } // namespace exegesis } // namespace llvm diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index 8d9ded1b2ac5e9c7871a9f3d5644dc371ca6a872..f1b2ff68e34319f6d979f2b100242f03bcbe1c09 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -282,6 +282,10 @@ static std::string getScopedName(unsigned Scope, const std::string &Name) { return ("pred:" + Twine(Scope) + ":" + Name).str(); } +static std::string getMangledRootDefName(StringRef DefOperandName) { + return ("DstI[" + DefOperandName + "]").str(); +} + //===- GlobalISelEmitter class --------------------------------------------===// static Expected getInstResultType(const TreePatternNode *Dst) { @@ -1499,8 +1503,13 @@ Expected GlobalISelEmitter::importExplicitDefRenderers( if (DstNumDefs == 0) return InsertPt; - for (unsigned I = 0; I < SrcNumDefs; ++I) - DstMIBuilder.addRenderer(DstI->Operands[I].Name); + for (unsigned I = 0; I < SrcNumDefs; ++I) { + std::string OpName = getMangledRootDefName(DstI->Operands[I].Name); + // CopyRenderer saves a StringRef, so cannot pass OpName itself - + // let's use a string with an appropriate lifetime. + StringRef PermanentRef = M.getOperandMatcher(OpName).getSymbolicName(); + DstMIBuilder.addRenderer(PermanentRef); + } // Some instructions have multiple defs, but are missing a type entry // (e.g. s_cc_out operands). @@ -2013,16 +2022,17 @@ Expected GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { const TypeSetByHwMode &VTy = Src->getExtType(I); const auto &DstIOperand = DstI.Operands[OpIdx]; - Record *DstIOpRec = DstIOperand.Rec; + PointerUnion MatchedRC = + DstIOperand.Rec; if (DstIName == "COPY_TO_REGCLASS") { - DstIOpRec = getInitValueAsRegClass(Dst->getChild(1)->getLeafValue()); + MatchedRC = getInitValueAsRegClass(Dst->getChild(1)->getLeafValue()); - if (DstIOpRec == nullptr) + if (MatchedRC.isNull()) return failedImport( "COPY_TO_REGCLASS operand #1 isn't a register class"); } else if (DstIName == "REG_SEQUENCE") { - DstIOpRec = getInitValueAsRegClass(Dst->getChild(0)->getLeafValue()); - if (DstIOpRec == nullptr) + MatchedRC = getInitValueAsRegClass(Dst->getChild(0)->getLeafValue()); + if (MatchedRC.isNull()) return failedImport("REG_SEQUENCE operand #0 isn't a register class"); } else if (DstIName == "EXTRACT_SUBREG") { auto InferredClass = inferRegClassFromPattern(Dst->getChild(0)); @@ -2032,7 +2042,7 @@ Expected GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { // We can assume that a subregister is in the same bank as it's super // register. - DstIOpRec = (*InferredClass)->getDef(); + MatchedRC = (*InferredClass)->getDef(); } else if (DstIName == "INSERT_SUBREG") { auto MaybeSuperClass = inferSuperRegisterClassForNode( VTy, Dst->getChild(0), Dst->getChild(2)); @@ -2042,34 +2052,30 @@ Expected GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { // Move to the next pattern here, because the register class we found // doesn't necessarily have a record associated with it. So, we can't // set DstIOpRec using this. - OperandMatcher &OM = InsnMatcher.getOperand(OpIdx); - OM.setSymbolicName(DstIOperand.Name); - M.defineOperand(OM.getSymbolicName(), OM); - OM.addPredicate(**MaybeSuperClass); - ++OpIdx; - continue; + MatchedRC = *MaybeSuperClass; } else if (DstIName == "SUBREG_TO_REG") { auto MaybeRegClass = inferSuperRegisterClass(VTy, Dst->getChild(2)); if (!MaybeRegClass) return failedImport( "Cannot infer register class for SUBREG_TO_REG operand #0"); - OperandMatcher &OM = InsnMatcher.getOperand(OpIdx); - OM.setSymbolicName(DstIOperand.Name); - M.defineOperand(OM.getSymbolicName(), OM); - OM.addPredicate(**MaybeRegClass); - ++OpIdx; - continue; - } else if (DstIOpRec->isSubClassOf("RegisterOperand")) - DstIOpRec = DstIOpRec->getValueAsDef("RegClass"); - else if (!DstIOpRec->isSubClassOf("RegisterClass")) + MatchedRC = *MaybeRegClass; + } else if (MatchedRC.get()->isSubClassOf("RegisterOperand")) + MatchedRC = MatchedRC.get()->getValueAsDef("RegClass"); + else if (!MatchedRC.get()->isSubClassOf("RegisterClass")) return failedImport("Dst MI def isn't a register class" + to_string(*Dst)); OperandMatcher &OM = InsnMatcher.getOperand(OpIdx); - OM.setSymbolicName(DstIOperand.Name); + // The operand names declared in the DstI instruction are unrelated to + // those used in pattern's source and destination DAGs, so mangle the + // former to prevent implicitly adding unexpected + // GIM_CheckIsSameOperand predicates by the defineOperand method. + OM.setSymbolicName(getMangledRootDefName(DstIOperand.Name)); M.defineOperand(OM.getSymbolicName(), OM); + if (MatchedRC.is()) + MatchedRC = &Target.getRegisterClass(MatchedRC.get()); OM.addPredicate( - Target.getRegisterClass(DstIOpRec)); + *MatchedRC.get()); ++OpIdx; } diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/GlobalISelMatchTable.cpp index 5a4d32a34e2bcb8419d9b83068faed8884c897d3..481f3f16e0136c696bc4bf869c0a56c3380c1e29 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp +++ b/llvm/utils/TableGen/GlobalISelMatchTable.cpp @@ -243,7 +243,7 @@ void MatchTable::emitDeclaration(raw_ostream &OS) const { if (I->Flags & MatchTableRecord::MTRF_Outdent) Indentation -= 2; } - OS << "};\n"; + OS << "}; // Size: " << (CurrentSize * 8) << " bytes\n"; } MatchTable MatchTable::buildTable(ArrayRef Rules, bool WithCoverage, diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h b/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h index c30198f11195c6808aefc46b8970bcfc8dec8418..7e952d6df30939d5539652f209e4ce67b6efd992 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h +++ b/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h @@ -20,7 +20,6 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include -#include namespace llvm { class CodeGenTarget; diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp index 3402809c4828c66356e511fd75ef5da6e9a14401..28604c5600bf4e6c6f5034b240af4f8648381c2a 100644 --- a/llvm/utils/TableGen/IntrinsicEmitter.cpp +++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp @@ -12,7 +12,6 @@ #include "CodeGenIntrinsics.h" #include "SequenceToOffsetTable.h" -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/utils/TableGen/X86DisassemblerTables.cpp b/llvm/utils/TableGen/X86DisassemblerTables.cpp index f879a9f5e4094c5da2807ab7d294e1eb3814e63c..959e0fda50b811ff9c626ebb39ba92d3ce513d6d 100644 --- a/llvm/utils/TableGen/X86DisassemblerTables.cpp +++ b/llvm/utils/TableGen/X86DisassemblerTables.cpp @@ -31,39 +31,47 @@ using namespace X86Disassembler; /// @param insnContext - The instruction class to transform to a string. /// @return - A statically-allocated string constant that contains the /// name of the instruction class. -static inline const char* stringForContext(InstructionContext insnContext) { +static inline const char *stringForContext(InstructionContext insnContext) { switch (insnContext) { default: llvm_unreachable("Unhandled instruction class"); -#define ENUM_ENTRY(n, r, d) case n: return #n; break; -#define ENUM_ENTRY_K_B(n, r, d) ENUM_ENTRY(n, r, d) ENUM_ENTRY(n##_K_B, r, d)\ - ENUM_ENTRY(n##_KZ, r, d) ENUM_ENTRY(n##_K, r, d) ENUM_ENTRY(n##_B, r, d)\ - ENUM_ENTRY(n##_KZ_B, r, d) - INSTRUCTION_CONTEXTS +#define ENUM_ENTRY(n, r, d) \ + case n: \ + return #n; \ + break; +#define ENUM_ENTRY_K_B(n, r, d) \ + ENUM_ENTRY(n, r, d) \ + ENUM_ENTRY(n##_K_B, r, d) ENUM_ENTRY(n##_KZ, r, d) ENUM_ENTRY(n##_K, r, d) \ + ENUM_ENTRY(n##_B, r, d) ENUM_ENTRY(n##_KZ_B, r, d) + INSTRUCTION_CONTEXTS #undef ENUM_ENTRY #undef ENUM_ENTRY_K_B } } /// stringForOperandType - Like stringForContext, but for OperandTypes. -static inline const char* stringForOperandType(OperandType type) { +static inline const char *stringForOperandType(OperandType type) { switch (type) { default: llvm_unreachable("Unhandled type"); -#define ENUM_ENTRY(i, d) case i: return #i; - TYPES +#define ENUM_ENTRY(i, d) \ + case i: \ + return #i; + TYPES #undef ENUM_ENTRY } } /// stringForOperandEncoding - like stringForContext, but for /// OperandEncodings. -static inline const char* stringForOperandEncoding(OperandEncoding encoding) { +static inline const char *stringForOperandEncoding(OperandEncoding encoding) { switch (encoding) { default: llvm_unreachable("Unhandled encoding"); -#define ENUM_ENTRY(i, d) case i: return #i; - ENCODINGS +#define ENUM_ENTRY(i, d) \ + case i: \ + return #i; + ENCODINGS #undef ENUM_ENTRY } } @@ -83,17 +91,17 @@ static inline bool inheritsFrom(InstructionContext child, switch (parent) { case IC: - return(inheritsFrom(child, IC_64BIT, AdSize64) || - (noPrefix && inheritsFrom(child, IC_OPSIZE, noPrefix)) || - inheritsFrom(child, IC_ADSIZE) || - (noPrefix && inheritsFrom(child, IC_XD, noPrefix)) || - (noPrefix && inheritsFrom(child, IC_XS, noPrefix))); + return (inheritsFrom(child, IC_64BIT, AdSize64) || + (noPrefix && inheritsFrom(child, IC_OPSIZE, noPrefix)) || + inheritsFrom(child, IC_ADSIZE) || + (noPrefix && inheritsFrom(child, IC_XD, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_XS, noPrefix))); case IC_64BIT: - return(inheritsFrom(child, IC_64BIT_REXW) || - (noPrefix && inheritsFrom(child, IC_64BIT_OPSIZE, noPrefix)) || - (!AdSize64 && inheritsFrom(child, IC_64BIT_ADSIZE)) || - (noPrefix && inheritsFrom(child, IC_64BIT_XD, noPrefix)) || - (noPrefix && inheritsFrom(child, IC_64BIT_XS, noPrefix))); + return (inheritsFrom(child, IC_64BIT_REXW) || + (noPrefix && inheritsFrom(child, IC_64BIT_OPSIZE, noPrefix)) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_ADSIZE)) || + (noPrefix && inheritsFrom(child, IC_64BIT_XD, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_64BIT_XS, noPrefix))); case IC_OPSIZE: return inheritsFrom(child, IC_64BIT_OPSIZE) || inheritsFrom(child, IC_OPSIZE_ADSIZE); @@ -118,20 +126,20 @@ static inline bool inheritsFrom(InstructionContext child, case IC_XS_ADSIZE: return inheritsFrom(child, IC_64BIT_XS_ADSIZE); case IC_64BIT_REXW: - return((noPrefix && inheritsFrom(child, IC_64BIT_REXW_XS, noPrefix)) || - (noPrefix && inheritsFrom(child, IC_64BIT_REXW_XD, noPrefix)) || - (noPrefix && inheritsFrom(child, IC_64BIT_REXW_OPSIZE, noPrefix)) || - (!AdSize64 && inheritsFrom(child, IC_64BIT_REXW_ADSIZE))); + return ((noPrefix && inheritsFrom(child, IC_64BIT_REXW_XS, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_64BIT_REXW_XD, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_64BIT_REXW_OPSIZE, noPrefix)) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_REXW_ADSIZE))); case IC_64BIT_OPSIZE: return inheritsFrom(child, IC_64BIT_REXW_OPSIZE) || (!AdSize64 && inheritsFrom(child, IC_64BIT_OPSIZE_ADSIZE)) || (!AdSize64 && inheritsFrom(child, IC_64BIT_REXW_ADSIZE)); case IC_64BIT_XD: - return(inheritsFrom(child, IC_64BIT_REXW_XD) || - (!AdSize64 && inheritsFrom(child, IC_64BIT_XD_ADSIZE))); + return (inheritsFrom(child, IC_64BIT_REXW_XD) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_XD_ADSIZE))); case IC_64BIT_XS: - return(inheritsFrom(child, IC_64BIT_REXW_XS) || - (!AdSize64 && inheritsFrom(child, IC_64BIT_XS_ADSIZE))); + return (inheritsFrom(child, IC_64BIT_REXW_XS) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_XS_ADSIZE))); case IC_64BIT_XD_OPSIZE: case IC_64BIT_XS_OPSIZE: return false; @@ -410,10 +418,8 @@ static inline bool inheritsFrom(InstructionContext child, (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_K_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_K_B)); case IC_EVEX_OPSIZE_K_B: - return (VEX_LIG && WIG && - inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K_B)) || - (VEX_LIG && WIG && - inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K_B)) || + return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K_B)) || + (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K_B)) || (WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_K_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_K_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_K_B)); @@ -436,10 +442,8 @@ static inline bool inheritsFrom(InstructionContext child, (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_KZ_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_KZ_B)); case IC_EVEX_OPSIZE_KZ_B: - return (VEX_LIG && WIG && - inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ_B)) || - (VEX_LIG && WIG && - inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ_B)) || + return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ_B)) || + (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ_B)) || (WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_KZ_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_KZ_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_KZ_B)); @@ -558,8 +562,8 @@ static inline bool inheritsFrom(InstructionContext child, case IC_EVEX_L2_W_OPSIZE_KZ_B: return false; default: - errs() << "Unknown instruction class: " << - stringForContext((InstructionContext)parent) << "\n"; + errs() << "Unknown instruction class: " + << stringForContext((InstructionContext)parent) << "\n"; llvm_unreachable("Unknown instruction class"); } } @@ -577,12 +581,12 @@ static inline bool outranks(InstructionContext upper, assert(lower < IC_max); #define ENUM_ENTRY(n, r, d) r, -#define ENUM_ENTRY_K_B(n, r, d) ENUM_ENTRY(n, r, d) \ - ENUM_ENTRY(n##_K_B, r, d) ENUM_ENTRY(n##_KZ_B, r, d) \ - ENUM_ENTRY(n##_KZ, r, d) ENUM_ENTRY(n##_K, r, d) ENUM_ENTRY(n##_B, r, d) - static int ranks[IC_max] = { - INSTRUCTION_CONTEXTS - }; +#define ENUM_ENTRY_K_B(n, r, d) \ + ENUM_ENTRY(n, r, d) \ + ENUM_ENTRY(n##_K_B, r, d) \ + ENUM_ENTRY(n##_KZ_B, r, d) ENUM_ENTRY(n##_KZ, r, d) ENUM_ENTRY(n##_K, r, d) \ + ENUM_ENTRY(n##_B, r, d) + static int ranks[IC_max] = {INSTRUCTION_CONTEXTS}; #undef ENUM_ENTRY #undef ENUM_ENTRY_K_B @@ -605,19 +609,19 @@ static ModRMDecisionType getDecisionType(ModRMDecision &decision) { satisfiesOneEntry = false; if (((index & 0xc0) == 0xc0) && - (decision.instructionIDs[index] != decision.instructionIDs[0xc0])) + (decision.instructionIDs[index] != decision.instructionIDs[0xc0])) satisfiesSplitRM = false; if (((index & 0xc0) != 0xc0) && - (decision.instructionIDs[index] != decision.instructionIDs[0x00])) + (decision.instructionIDs[index] != decision.instructionIDs[0x00])) satisfiesSplitRM = false; - if (((index & 0xc0) == 0xc0) && - (decision.instructionIDs[index] != decision.instructionIDs[index&0xf8])) + if (((index & 0xc0) == 0xc0) && (decision.instructionIDs[index] != + decision.instructionIDs[index & 0xf8])) satisfiesSplitReg = false; - if (((index & 0xc0) != 0xc0) && - (decision.instructionIDs[index] != decision.instructionIDs[index&0x38])) + if (((index & 0xc0) != 0xc0) && (decision.instructionIDs[index] != + decision.instructionIDs[index & 0x38])) satisfiesSplitMisc = false; } @@ -642,11 +646,13 @@ static ModRMDecisionType getDecisionType(ModRMDecision &decision) { /// @param dt - The decision type. /// @return - A pointer to the statically-allocated string (e.g., /// "MODRM_ONEENTRY" for MODRM_ONEENTRY). -static const char* stringForDecisionType(ModRMDecisionType dt) { -#define ENUM_ENTRY(n) case n: return #n; +static const char *stringForDecisionType(ModRMDecisionType dt) { +#define ENUM_ENTRY(n) \ + case n: \ + return #n; switch (dt) { - default: - llvm_unreachable("Unknown decision type"); + default: + llvm_unreachable("Unknown decision type"); MODRMTYPES }; #undef ENUM_ENTRY @@ -659,8 +665,7 @@ DisassemblerTables::DisassemblerTables() { HasConflicts = false; } -DisassemblerTables::~DisassemblerTables() { -} +DisassemblerTables::~DisassemblerTables() {} void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2, unsigned &i1, unsigned &i2, @@ -678,31 +683,31 @@ void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2, std::vector ModRMDecision; switch (dt) { - default: - llvm_unreachable("Unknown decision type"); - case MODRM_ONEENTRY: - ModRMDecision.push_back(decision.instructionIDs[0]); - break; - case MODRM_SPLITRM: - ModRMDecision.push_back(decision.instructionIDs[0x00]); - ModRMDecision.push_back(decision.instructionIDs[0xc0]); - break; - case MODRM_SPLITREG: - for (unsigned index = 0; index < 64; index += 8) - ModRMDecision.push_back(decision.instructionIDs[index]); - for (unsigned index = 0xc0; index < 256; index += 8) - ModRMDecision.push_back(decision.instructionIDs[index]); - break; - case MODRM_SPLITMISC: - for (unsigned index = 0; index < 64; index += 8) - ModRMDecision.push_back(decision.instructionIDs[index]); - for (unsigned index = 0xc0; index < 256; ++index) - ModRMDecision.push_back(decision.instructionIDs[index]); - break; - case MODRM_FULL: - for (unsigned short InstructionID : decision.instructionIDs) - ModRMDecision.push_back(InstructionID); - break; + default: + llvm_unreachable("Unknown decision type"); + case MODRM_ONEENTRY: + ModRMDecision.push_back(decision.instructionIDs[0]); + break; + case MODRM_SPLITRM: + ModRMDecision.push_back(decision.instructionIDs[0x00]); + ModRMDecision.push_back(decision.instructionIDs[0xc0]); + break; + case MODRM_SPLITREG: + for (unsigned index = 0; index < 64; index += 8) + ModRMDecision.push_back(decision.instructionIDs[index]); + for (unsigned index = 0xc0; index < 256; index += 8) + ModRMDecision.push_back(decision.instructionIDs[index]); + break; + case MODRM_SPLITMISC: + for (unsigned index = 0; index < 64; index += 8) + ModRMDecision.push_back(decision.instructionIDs[index]); + for (unsigned index = 0xc0; index < 256; ++index) + ModRMDecision.push_back(decision.instructionIDs[index]); + break; + case MODRM_FULL: + for (unsigned short InstructionID : decision.instructionIDs) + ModRMDecision.push_back(InstructionID); + break; } unsigned &EntryNumber = ModRMTable[ModRMDecision]; @@ -722,23 +727,23 @@ void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2, o2 << "{" << stringForDecisionType(dt) << ", " << EntryNumber << "}"; switch (dt) { - default: - llvm_unreachable("Unknown decision type"); - case MODRM_ONEENTRY: - sEntryNumber += 1; - break; - case MODRM_SPLITRM: - sEntryNumber += 2; - break; - case MODRM_SPLITREG: - sEntryNumber += 16; - break; - case MODRM_SPLITMISC: - sEntryNumber += 8 + 64; - break; - case MODRM_FULL: - sEntryNumber += 256; - break; + default: + llvm_unreachable("Unknown decision type"); + case MODRM_ONEENTRY: + sEntryNumber += 1; + break; + case MODRM_SPLITRM: + sEntryNumber += 2; + break; + case MODRM_SPLITREG: + sEntryNumber += 16; + break; + case MODRM_SPLITMISC: + sEntryNumber += 8 + 64; + break; + case MODRM_FULL: + sEntryNumber += 256; + break; } // We assume that the index can fit into uint16_t. @@ -791,8 +796,9 @@ void DisassemblerTables::emitContextDecision(raw_ostream &o1, raw_ostream &o2, unsigned &i1, unsigned &i2, unsigned &ModRMTableNum, ContextDecision &decision, - const char* name) const { - o2.indent(i2) << "static const struct ContextDecision " << name << " = {{/* opcodeDecisions */\n"; + const char *name) const { + o2.indent(i2) << "static const struct ContextDecision " << name + << " = {{/* opcodeDecisions */\n"; i2++; for (unsigned index = 0; index < IC_max; ++index) { @@ -815,8 +821,8 @@ void DisassemblerTables::emitInstructionInfo(raw_ostream &o, o << "static const struct OperandSpecifier x86OperandSets[][" << X86_MAX_OPERANDS << "] = {\n"; - typedef SmallVector, - X86_MAX_OPERANDS> OperandListTy; + typedef SmallVector, X86_MAX_OPERANDS> + OperandListTy; std::map OperandSets; unsigned OperandSetNum = 0; @@ -829,14 +835,15 @@ void DisassemblerTables::emitInstructionInfo(raw_ostream &o, OperandList.push_back(std::make_pair(Encoding, Type)); } unsigned &N = OperandSets[OperandList]; - if (N != 0) continue; + if (N != 0) + continue; N = ++OperandSetNum; o << " { /* " << (OperandSetNum - 1) << " */\n"; for (unsigned i = 0, e = OperandList.size(); i != e; ++i) { const char *Encoding = stringForOperandEncoding(OperandList[i].first); - const char *Type = stringForOperandType(OperandList[i].second); + const char *Type = stringForOperandType(OperandList[i].second); o << " { " << Encoding << ", " << Type << " },\n"; } o << " },\n"; @@ -871,8 +878,8 @@ void DisassemblerTables::emitInstructionInfo(raw_ostream &o, } void DisassemblerTables::emitContextTable(raw_ostream &o, unsigned &i) const { - o.indent(i * 2) << "static const uint8_t " CONTEXTS_STR - "[" << ATTR_max << "] = {\n"; + o.indent(i * 2) << "static const uint8_t " CONTEXTS_STR "[" << ATTR_max + << "] = {\n"; i++; for (unsigned index = 0; index < ATTR_max; ++index) { @@ -908,8 +915,7 @@ void DisassemblerTables::emitContextTable(raw_ostream &o, unsigned &i) const { if (index & ATTR_EVEXB) o << "_B"; } - } - else if ((index & ATTR_64BIT) && (index & ATTR_REX2)) + } else if ((index & ATTR_64BIT) && (index & ATTR_REX2)) o << "IC_64BIT_REX2"; else if ((index & ATTR_64BIT) && (index & ATTR_REXW) && (index & ATTR_XS)) o << "IC_64BIT_REXW_XS"; @@ -977,12 +983,15 @@ void DisassemblerTables::emitContextDecisions(raw_ostream &o1, raw_ostream &o2, unsigned &ModRMTableNum) const { emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[0], ONEBYTE_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[1], TWOBYTE_STR); - emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[2], THREEBYTE38_STR); - emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[3], THREEBYTE3A_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[2], + THREEBYTE38_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[3], + THREEBYTE3A_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[4], XOP8_MAP_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[5], XOP9_MAP_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[6], XOPA_MAP_STR); - emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[7], THREEDNOW_MAP_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[7], + THREEDNOW_MAP_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[8], MAP4_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[9], MAP5_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[10], MAP6_STR); @@ -1026,24 +1035,22 @@ void DisassemblerTables::emit(raw_ostream &o) const { o << "\n"; } -void DisassemblerTables::setTableFields(ModRMDecision &decision, - const ModRMFilter &filter, - InstrUID uid, - uint8_t opcode) { +void DisassemblerTables::setTableFields(ModRMDecision &decision, + const ModRMFilter &filter, InstrUID uid, + uint8_t opcode) { for (unsigned index = 0; index < 256; ++index) { if (filter.accepts(index)) { if (decision.instructionIDs[index] == uid) continue; if (decision.instructionIDs[index] != 0) { - InstructionSpecifier &newInfo = - InstructionSpecifiers[uid]; + InstructionSpecifier &newInfo = InstructionSpecifiers[uid]; InstructionSpecifier &previousInfo = - InstructionSpecifiers[decision.instructionIDs[index]]; + InstructionSpecifiers[decision.instructionIDs[index]]; - if(previousInfo.name == "NOOP" && (newInfo.name == "XCHG16ar" || - newInfo.name == "XCHG32ar" || - newInfo.name == "XCHG64ar")) + if (previousInfo.name == "NOOP" && + (newInfo.name == "XCHG16ar" || newInfo.name == "XCHG32ar" || + newInfo.name == "XCHG64ar")) continue; // special case for XCHG*ar and NOOP if (outranks(previousInfo.insnContext, newInfo.insnContext)) @@ -1065,16 +1072,10 @@ void DisassemblerTables::setTableFields(ModRMDecision &decision, } } -void DisassemblerTables::setTableFields(OpcodeType type, - InstructionContext insnContext, - uint8_t opcode, - const ModRMFilter &filter, - InstrUID uid, - bool is32bit, - bool noPrefix, - bool ignoresVEX_L, - bool ignoresW, - unsigned addressSize) { +void DisassemblerTables::setTableFields( + OpcodeType type, InstructionContext insnContext, uint8_t opcode, + const ModRMFilter &filter, InstrUID uid, bool is32bit, bool noPrefix, + bool ignoresVEX_L, bool ignoresW, unsigned addressSize) { ContextDecision &decision = *Tables[type]; for (unsigned index = 0; index < IC_max; ++index) { @@ -1087,8 +1088,6 @@ void DisassemblerTables::setTableFields(OpcodeType type, InstructionSpecifiers[uid].insnContext, noPrefix, ignoresVEX_L, ignoresW, adSize64)) setTableFields(decision.opcodeDecisions[index].modRMDecisions[opcode], - filter, - uid, - opcode); + filter, uid, opcode); } } diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py old mode 100644 new mode 100755 index 8876ba7a14debece15c321f2d0eaa07e0d17493b..697a27ab82a750d1636271f8020408cee6de43c0 --- a/llvm/utils/git/code-format-helper.py +++ b/llvm/utils/git/code-format-helper.py @@ -1,21 +1,58 @@ #!/usr/bin/env python3 # -# ====- code-format-helper, runs code formatters from the ci --*- python -*--==# +# ====- code-format-helper, runs code formatters from the ci or in a hook --*- python -*--==# # # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # -# ==-------------------------------------------------------------------------==# +# ==--------------------------------------------------------------------------------------==# import argparse import os import subprocess import sys -from functools import cached_property +from typing import List, Optional -import github -from github import IssueComment, PullRequest +""" +This script is run by GitHub actions to ensure that the code in PR's conform to +the coding style of LLVM. It can also be installed as a pre-commit git hook to +check the coding style before submitting it. The canonical source of this script +is in the LLVM source tree under llvm/utils/git. + +For C/C++ code it uses clang-format and for Python code it uses darker (which +in turn invokes black). + +You can learn more about the LLVM coding style on llvm.org: +https://llvm.org/docs/CodingStandards.html + +You can install this script as a git hook by symlinking it to the .git/hooks +directory: + +ln -s $(pwd)/llvm/utils/git/code-format-helper.py .git/hooks/pre-commit + +You can control the exact path to clang-format or darker with the following +environment variables: $CLANG_FORMAT_PATH and $DARKER_FORMAT_PATH. +""" + + +class FormatArgs: + start_rev: str = None + end_rev: str = None + repo: str = None + changed_files: List[str] = [] + token: str = None + verbose: bool = True + issue_number: int = 0 + + def __init__(self, args: argparse.Namespace = None) -> None: + if not args is None: + self.start_rev = args.start_rev + self.end_rev = args.end_rev + self.repo = args.repo + self.token = args.token + self.changed_files = args.changed_files + self.issue_number = args.issue_number class FormatHelper: @@ -31,9 +68,10 @@ class FormatHelper: def instructions(self) -> str: raise NotImplementedError() - def format_run( - self, changed_files: list[str], args: argparse.Namespace - ) -> str | None: + def has_tool(self) -> bool: + raise NotImplementedError() + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: raise NotImplementedError() def pr_comment_text_for_diff(self, diff: str) -> str: @@ -63,17 +101,18 @@ View the diff from {self.name} here. """ - def find_comment( - self, pr: PullRequest.PullRequest - ) -> IssueComment.IssueComment | None: + # TODO: any type should be replaced with the correct github type, but it requires refactoring to + # not require the github module to be installed everywhere. + def find_comment(self, pr: any) -> any: for comment in pr.as_issue().get_comments(): if self.comment_tag in comment.body: return comment return None - def update_pr( - self, comment_text: str, args: argparse.Namespace, create_new: bool - ) -> None: + def update_pr(self, comment_text: str, args: FormatArgs, create_new: bool) -> None: + import github + from github import IssueComment, PullRequest + repo = github.Github(args.token).get_repo(args.repo) pr = repo.get_issue(args.issue_number).as_pull_request() @@ -85,17 +124,25 @@ View the diff from {self.name} here. elif create_new: pr.as_issue().create_comment(comment_text) - def run(self, changed_files: list[str], args: argparse.Namespace) -> bool: + def run(self, changed_files: List[str], args: FormatArgs) -> bool: diff = self.format_run(changed_files, args) + should_update_gh = args.token is not None and args.repo is not None + if diff is None: - comment_text = f""" -:white_check_mark: With the latest revision this PR passed the {self.friendly_name}. -""" - self.update_pr(comment_text, args, create_new=False) + if should_update_gh: + comment_text = f""" + :white_check_mark: With the latest revision this PR passed the {self.friendly_name}. + """ + self.update_pr(comment_text, args, create_new=False) return True elif len(diff) > 0: - comment_text = self.pr_comment_text_for_diff(diff) - self.update_pr(comment_text, args, create_new=True) + if should_update_gh: + comment_text = self.pr_comment_text_for_diff(diff) + self.update_pr(comment_text, args, create_new=True) + else: + print( + f"Warning: {self.friendly_name}, {self.name} detected some issues with your code formatting..." + ) return False else: # The formatter failed but didn't output a diff (e.g. some sort of @@ -118,7 +165,7 @@ class ClangFormatHelper(FormatHelper): def should_include_extensionless_file(self, path: str) -> bool: return path.startswith("libcxx/include") - def filter_changed_files(self, changed_files: list[str]) -> list[str]: + def filter_changed_files(self, changed_files: List[str]) -> List[str]: filtered_files = [] for path in changed_files: _, ext = os.path.splitext(path) @@ -128,32 +175,49 @@ class ClangFormatHelper(FormatHelper): filtered_files.append(path) return filtered_files - def format_run( - self, changed_files: list[str], args: argparse.Namespace - ) -> str | None: + @property + def clang_fmt_path(self) -> str: + if "CLANG_FORMAT_PATH" in os.environ: + return os.environ["CLANG_FORMAT_PATH"] + return "git-clang-format" + + def has_tool(self) -> bool: + cmd = [self.clang_fmt_path, "-h"] + proc = None + try: + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except: + return False + return proc.returncode == 0 + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: cpp_files = self.filter_changed_files(changed_files) if not cpp_files: return None - cf_cmd = [ - "git-clang-format", - "--diff", - args.start_rev, - args.end_rev, - "--", - ] + cpp_files - print(f"Running: {' '.join(cf_cmd)}") + + cf_cmd = [self.clang_fmt_path, "--diff"] + + if args.start_rev and args.end_rev: + cf_cmd.append(args.start_rev) + cf_cmd.append(args.end_rev) + + cf_cmd.append("--") + cf_cmd += cpp_files + + if args.verbose: + print(f"Running: {' '.join(cf_cmd)}") self.cf_cmd = cf_cmd - proc = subprocess.run(cf_cmd, capture_output=True) + proc = subprocess.run(cf_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) sys.stdout.write(proc.stderr.decode("utf-8")) if proc.returncode != 0: # formatting needed, or the command otherwise failed - print(f"error: {self.name} exited with code {proc.returncode}") - # Print the diff in the log so that it is viewable there - print(proc.stdout.decode("utf-8")) + if args.verbose: + print(f"error: {self.name} exited with code {proc.returncode}") + # Print the diff in the log so that it is viewable there + print(proc.stdout.decode("utf-8")) return proc.stdout.decode("utf-8") else: - sys.stdout.write(proc.stdout.decode("utf-8")) return None @@ -165,7 +229,7 @@ class DarkerFormatHelper(FormatHelper): def instructions(self) -> str: return " ".join(self.darker_cmd) - def filter_changed_files(self, changed_files: list[str]) -> list[str]: + def filter_changed_files(self, changed_files: List[str]) -> List[str]: filtered_files = [] for path in changed_files: name, ext = os.path.splitext(path) @@ -174,29 +238,48 @@ class DarkerFormatHelper(FormatHelper): return filtered_files - def format_run( - self, changed_files: list[str], args: argparse.Namespace - ) -> str | None: + @property + def darker_fmt_path(self) -> str: + if "DARKER_FORMAT_PATH" in os.environ: + return os.environ["DARKER_FORMAT_PATH"] + return "darker" + + def has_tool(self) -> bool: + cmd = [self.darker_fmt_path, "--version"] + proc = None + try: + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except: + return False + return proc.returncode == 0 + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: py_files = self.filter_changed_files(changed_files) if not py_files: return None darker_cmd = [ - "darker", + self.darker_fmt_path, "--check", "--diff", - "-r", - f"{args.start_rev}..{args.end_rev}", - ] + py_files - print(f"Running: {' '.join(darker_cmd)}") + ] + if args.start_rev and args.end_rev: + darker_cmd += ["-r", f"{args.start_rev}...{args.end_rev}"] + darker_cmd += py_files + if args.verbose: + print(f"Running: {' '.join(darker_cmd)}") self.darker_cmd = darker_cmd - proc = subprocess.run(darker_cmd, capture_output=True) - sys.stdout.write(proc.stderr.decode("utf-8")) + proc = subprocess.run( + darker_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if args.verbose: + sys.stdout.write(proc.stderr.decode("utf-8")) if proc.returncode != 0: # formatting needed, or the command otherwise failed - print(f"error: {self.name} exited with code {proc.returncode}") - # Print the diff in the log so that it is viewable there - print(proc.stdout.decode("utf-8")) + if args.verbose: + print(f"error: {self.name} exited with code {proc.returncode}") + # Print the diff in the log so that it is viewable there + print(proc.stdout.decode("utf-8")) return proc.stdout.decode("utf-8") else: sys.stdout.write(proc.stdout.decode("utf-8")) @@ -205,7 +288,39 @@ class DarkerFormatHelper(FormatHelper): ALL_FORMATTERS = (DarkerFormatHelper(), ClangFormatHelper()) + +def hook_main(): + # fill out args + args = FormatArgs() + args.verbose = False + + # find the changed files + cmd = ["git", "diff", "--cached", "--name-only", "--diff-filter=d"] + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output = proc.stdout.decode("utf-8") + for line in output.splitlines(): + args.changed_files.append(line) + + failed_fmts = [] + for fmt in ALL_FORMATTERS: + if fmt.has_tool(): + if not fmt.run(args.changed_files, args): + failed_fmts.append(fmt.name) + else: + print(f"Couldn't find {fmt.name}, can't check " + fmt.friendly_name.lower()) + + if len(failed_fmts) > 0: + sys.exit(1) + + sys.exit(0) + + if __name__ == "__main__": + script_path = os.path.abspath(__file__) + if ".git/hooks" in script_path: + hook_main() + sys.exit(0) + parser = argparse.ArgumentParser() parser.add_argument( "--token", type=str, required=True, help="GitHub authentiation token" @@ -232,7 +347,7 @@ if __name__ == "__main__": help="Comma separated list of files that has been changed", ) - args = parser.parse_args() + args = FormatArgs(parser.parse_args()) changed_files = [] if args.changed_files: diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/hicpp/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/hicpp/BUILD.gn index cd886d89afcf75a3855c3827e7228238fdc67803..cb0e823b3acb47b8e20e8d55b95cb34fd9403643 100644 --- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/hicpp/BUILD.gn +++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/hicpp/BUILD.gn @@ -21,6 +21,7 @@ static_library("hicpp") { sources = [ "ExceptionBaseclassCheck.cpp", "HICPPTidyModule.cpp", + "IgnoredRemoveResultCheck.cpp", "MultiwayPathsCoveredCheck.cpp", "NoAssemblerCheck.cpp", "SignedBitwiseCheck.cpp", diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn index 64d38360e2500b79add6b851c5fd614e7e555684..1d3fc6cfdfaaa6d77dc02d9a9253eb6a8fe5b608 100644 --- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn @@ -1,9 +1,9 @@ static_library("Debugging") { output_name = "LLVMOrcDebugging" deps = [ + "//llvm/lib/DebugInfo/DWARF", "//llvm/lib/ExecutionEngine/Orc", "//llvm/lib/ExecutionEngine/Orc/Shared", - "//llvm/lib/DebugInfo/DWARF", "//llvm/lib/Support", "//llvm/lib/TargetParser", ] @@ -11,6 +11,7 @@ static_library("Debugging") { "DebugInfoSupport.cpp", "DebuggerSupport.cpp", "DebuggerSupportPlugin.cpp", + "LLJITUtilsCBindings.cpp", "PerfSupportPlugin.cpp", ] if (current_os == "linux") { diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn index 2e5b7e03bd65d0c0a2e02ab00c9dcaa7e02f58a2..668512ecba88e32b39d99190ebfb919abd4f7fc0 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn @@ -80,6 +80,10 @@ static_library("LLVMBPFCodeGen") { "BPFSubtarget.cpp", "BPFTargetMachine.cpp", "BTFDebug.cpp", + "GISel/BPFCallLowering.cpp", + "GISel/BPFInstructionSelector.cpp", + "GISel/BPFLegalizerInfo.cpp", + "GISel/BPFRegisterBankInfo.cpp", ] } diff --git a/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn index c258b523228002e2d012b74fc313d31521405ffb..47c7bb3d97d1c55d30f5129cbd670b3de575da50 100644 --- a/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn @@ -4,6 +4,7 @@ unittest("OrcJITTests") { deps = [ "//llvm/lib/ExecutionEngine", "//llvm/lib/ExecutionEngine/Orc", + "//llvm/lib/ExecutionEngine/Orc/Debugging", "//llvm/lib/ExecutionEngine/Orc/Shared", "//llvm/lib/ExecutionEngine/RuntimeDyld", "//llvm/lib/IR", diff --git a/mlir/cmake/modules/MLIRConfig.cmake.in b/mlir/cmake/modules/MLIRConfig.cmake.in index cd8cec5dedf3c1729aaa1468838a00cc66d4ae13..d4da3cd98cce9816e30f45b96e6b5dbf00ad33d6 100644 --- a/mlir/cmake/modules/MLIRConfig.cmake.in +++ b/mlir/cmake/modules/MLIRConfig.cmake.in @@ -13,6 +13,7 @@ set(MLIR_TABLEGEN_EXE "@MLIR_CONFIG_TABLEGEN_EXE@") set(MLIR_PDLL_TABLEGEN_EXE "@MLIR_CONFIG_PDLL_TABLEGEN_EXE@") set(MLIR_INSTALL_AGGREGATE_OBJECTS "@MLIR_INSTALL_AGGREGATE_OBJECTS@") set(MLIR_ENABLE_BINDINGS_PYTHON "@MLIR_ENABLE_BINDINGS_PYTHON@") +set(MLIR_ENABLE_EXECUTION_ENGINE "@MLIR_ENABLE_EXECUTION_ENGINE@") # For mlir_tablegen() set(MLIR_INCLUDE_DIR "@MLIR_INCLUDE_DIR@") diff --git a/mlir/include/mlir-c/IR.h b/mlir/include/mlir-c/IR.h index 413eaa6aa3fe0ece474c9fce68ec8d665729abc5..82da511f807a3460ebdce56ff4de6fe7c79daa78 100644 --- a/mlir/include/mlir-c/IR.h +++ b/mlir/include/mlir-c/IR.h @@ -940,6 +940,9 @@ MLIR_CAPI_EXPORTED void mlirValueReplaceAllUsesOfWith(MlirValue of, /// Returns whether the op operand is null. MLIR_CAPI_EXPORTED bool mlirOpOperandIsNull(MlirOpOperand opOperand); +/// Returns the value of an op operand. +MLIR_CAPI_EXPORTED MlirValue mlirOpOperandGetValue(MlirOpOperand opOperand); + /// Returns the owner operation of an op operand. MLIR_CAPI_EXPORTED MlirOperation mlirOpOperandGetOwner(MlirOpOperand opOperand); diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td index f9578cf37d5d76844160582754a9e67d5cd93d5b..c638646b9c3277d19aaf7e0942c79116ca0f2859 100644 --- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td +++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td @@ -16,6 +16,7 @@ include "mlir/Dialect/Arith/IR/ArithBase.td" include "mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.td" include "mlir/Interfaces/ControlFlowInterfaces.td" +include "mlir/Interfaces/InferTypeOpInterface.td" include "mlir/Interfaces/LoopLikeInterface.td" include "mlir/Interfaces/SideEffectInterfaces.td" @@ -63,10 +64,6 @@ def AffineApplyOp : Affine_Op<"apply", [Pure]> { // has a constant builder. That way we wouldn't need to explicitly specify the // result types here. let builders = [ - OpBuilder<(ins "AffineMap":$map, "ValueRange":$mapOperands), - [{ - build($_builder, $_state, $_builder.getIndexType(), map, mapOperands); - }]>, OpBuilder<(ins "ArrayRef ":$exprList,"ValueRange":$mapOperands), [{ build($_builder, $_state, $_builder.getIndexType(), @@ -541,13 +538,6 @@ class AffineMinMaxOpBase traits = []> : let arguments = (ins AffineMapAttr:$map, Variadic:$operands); let results = (outs Index); - let builders = [ - OpBuilder<(ins "AffineMap":$affineMap, "ValueRange":$mapOperands), - [{ - build($_builder, $_state, $_builder.getIndexType(), affineMap, mapOperands); - }]> - ]; - let extraClassDeclaration = [{ static StringRef getMapAttrStrName() { return "map"; } AffineMap getAffineMap() { return getMap(); } @@ -1068,7 +1058,7 @@ def AffineVectorStoreOp : AffineStoreOpBase<"vector_store"> { //===----------------------------------------------------------------------===// def AffineDelinearizeIndexOp : Affine_Op<"delinearize_index", - [Pure]> { + [Pure, DeclareOpInterfaceMethods]> { let summary = "delinearize an index"; let description = [{ The `affine.delinearize_index` operation takes a single index value and diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td index 02e0b7980c1077d696891e6733cd56f929e06884..6d133d69dd0faffb1beae18a63966617ae284035 100644 --- a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td +++ b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td @@ -195,6 +195,12 @@ def Arith_ConstantOp : Op { let summary = "integer addition operation"; let description = [{ + Performs N-bit addition on the operands. The operands are interpreted as + unsigned bitvectors. The result is represented by a bitvector containing the + mathematical value of the addition modulo 2^n, where `n` is the bitwidth. + Because `arith` integers use a two's complement representation, this operation + is applicable on both signed and unsigned integer operands. + The `addi` operation takes two operands and returns one result, each of these is required to be the same type. This type may be an integer scalar type, a vector whose element type is integer, or a tensor of integers. It has no @@ -230,7 +236,7 @@ def Arith_AddUIExtendedOp : Arith_Op<"addui_extended", [Pure, Commutative, let description = [{ Performs (N+1)-bit addition on zero-extended operands. Returns two results: the N-bit sum (same type as both operands), and the overflow bit - (boolean-like), where`1` indicates unsigned addition overflow, while `0` + (boolean-like), where `1` indicates unsigned addition overflow, while `0` indicates no overflow. Example: @@ -276,6 +282,18 @@ def Arith_SubIOp : Arith_TotalIntBinaryOp<"subi"> { let summary = [{ Integer subtraction operation. }]; + let description = [{ + Performs N-bit subtraction on the operands. The operands are interpreted as unsigned + bitvectors. The result is represented by a bitvector containing the mathematical + value of the subtraction modulo 2^n, where `n` is the bitwidth. Because `arith` + integers use a two's complement representation, this operation is applicable on + both signed and unsigned integer operands. + + The `subi` operation takes two operands and returns one result, each of + these is required to be the same type. This type may be an integer scalar type, + a vector whose element type is integer, or a tensor of integers. It has no + standard attributes. + }]; let hasFolder = 1; let hasCanonicalizer = 1; } @@ -288,6 +306,18 @@ def Arith_MulIOp : Arith_TotalIntBinaryOp<"muli", [Commutative]> { let summary = [{ Integer multiplication operation. }]; + let description = [{ + Performs N-bit multiplication on the operands. The operands are interpreted as + unsigned bitvectors. The result is represented by a bitvector containing the + mathematical value of the multiplication modulo 2^n, where `n` is the bitwidth. + Because `arith` integers use a two's complement representation, this operation is + applicable on both signed and unsigned integer operands. + + The `muli` operation takes two operands and returns one result, each of + these is required to be the same type. This type may be an integer scalar type, + a vector whose element type is integer, or a tensor of integers. It has no + standard attributes. + }]; let hasFolder = 1; let hasCanonicalizer = 1; } diff --git a/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td b/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td index 7b9c74e0b8f60e79c8fc537a5c936000757218ed..02238f0a18bab9825d6073423c13ed8e5451b3b4 100644 --- a/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td +++ b/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td @@ -87,8 +87,9 @@ def TileAllocation let summary = "Allocate SME tiles"; let description = [{ This pass does tile allocation for SME "virtual tiles". It is run at the - 'func.func' op level, replacing 'arm_sme.get_tile_id' ops with (i32) tile - ids. An error will be emitted when there's no tiles left. + 'func.func' op level, and assigns tile IDs (via an attribute) to all ops + that implement the `ArmSMETileOpInterface`. An error will be emitted when + there's no tiles left. }]; let constructor = "mlir::arm_sme::createTileAllocationPass()"; let dependentDialects = ["func::FuncDialect"]; diff --git a/mlir/include/mlir/Dialect/Complex/IR/ComplexOps.td b/mlir/include/mlir/Dialect/Complex/IR/ComplexOps.td index ada6c14b5b71354b45a19722f4fa6033f177e043..e19d714cadf8a9496ac45741b34a85ee624aaf4d 100644 --- a/mlir/include/mlir/Dialect/Complex/IR/ComplexOps.td +++ b/mlir/include/mlir/Dialect/Complex/IR/ComplexOps.td @@ -145,7 +145,7 @@ def ConstantOp : Complex_Op<"constant", [ }]; let arguments = (ins ArrayAttr:$value); - let results = (outs Complex:$complex); + let results = (outs AnyComplex:$complex); let assemblyFormat = "$value attr-dict `:` type($complex)"; let hasFolder = 1; diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td index 3b5984498cf83fe8635a0cfab71554fc60bd8127..a7b269eb41ee2e3660f8b5a77d93c54d7568572e 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td @@ -428,6 +428,29 @@ def DISubprogramFlags : I32BitEnumAttr< let printBitEnumPrimaryGroups = 1; } +//===----------------------------------------------------------------------===// +// IntegerOverflowFlags +//===----------------------------------------------------------------------===// + +def IOFnone : I32BitEnumAttrCaseNone<"none">; +def IOFnsw : I32BitEnumAttrCaseBit<"nsw", 0>; +def IOFnuw : I32BitEnumAttrCaseBit<"nuw", 1>; + +def IntegerOverflowFlags : I32BitEnumAttr< + "IntegerOverflowFlags", + "LLVM integer overflow flags", + [IOFnone, IOFnsw, IOFnuw]> { + let separator = ", "; + let cppNamespace = "::mlir::LLVM"; + let genSpecializedAttr = 0; + let printBitEnumPrimaryGroups = 1; +} + +def LLVM_IntegerOverflowFlagsAttr : + EnumAttr { + let assemblyFormat = "`<` $value `>`"; +} + //===----------------------------------------------------------------------===// // FastmathFlags //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td index c5d65f792254e607f2e57a5d01ffb7bec785ff1a..81589eaf5fd0a4382771a31c33b7d3f201b43604 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td @@ -48,6 +48,63 @@ def FastmathFlagsInterface : OpInterface<"FastmathFlagsInterface"> { ]; } +def IntegerOverflowFlagsInterface : OpInterface<"IntegerOverflowFlagsInterface"> { + let description = [{ + Access to op integer overflow flags. + }]; + + let cppNamespace = "::mlir::LLVM"; + + let methods = [ + InterfaceMethod< + /*desc=*/ "Returns an IntegerOverflowFlagsAttr attribute for the operation", + /*returnType=*/ "IntegerOverflowFlagsAttr", + /*methodName=*/ "getOverflowAttr", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + auto op = cast(this->getOperation()); + return op.getOverflowFlagsAttr(); + }] + >, + InterfaceMethod< + /*desc=*/ "Returns whether the operation has the No Unsigned Wrap keyword", + /*returnType=*/ "bool", + /*methodName=*/ "hasNoUnsignedWrap", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + auto op = cast(this->getOperation()); + IntegerOverflowFlags flags = op.getOverflowFlagsAttr().getValue(); + return bitEnumContainsAll(flags, IntegerOverflowFlags::nuw); + }] + >, + InterfaceMethod< + /*desc=*/ "Returns whether the operation has the No Signed Wrap keyword", + /*returnType=*/ "bool", + /*methodName=*/ "hasNoSignedWrap", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + auto op = cast(this->getOperation()); + IntegerOverflowFlags flags = op.getOverflowFlagsAttr().getValue(); + return bitEnumContainsAll(flags, IntegerOverflowFlags::nsw); + }] + >, + StaticInterfaceMethod< + /*desc=*/ [{Returns the name of the IntegerOveflowFlagsAttr attribute + for the operation}], + /*returnType=*/ "StringRef", + /*methodName=*/ "getIntegerOverflowAttrName", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + return "overflowFlags"; + }] + > + ]; +} + def BranchWeightOpInterface : OpInterface<"BranchWeightOpInterface"> { let description = [{ An interface for operations that can carry branch weights metadata. It diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index 92460fa06f530db93b5b5bac12403854dc280ccd..d7690b84807f6e308fa34da806abb561fa754cd0 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -55,6 +55,26 @@ class LLVM_IntArithmeticOp($_location, $lhs, $rhs); }]; } +class LLVM_IntArithmeticOpWithOverflowFlag traits = []> : + LLVM_ArithmeticOpBase], traits)> { + dag iofArg = ( + ins DefaultValuedAttr:$overflowFlags); + let arguments = !con(commonArgs, iofArg); + string mlirBuilder = [{ + auto op = $_builder.create<$_qualCppClassName>($_location, $lhs, $rhs); + moduleImport.setIntegerOverflowFlagsAttr(inst, op); + $res = op; + }]; + let assemblyFormat = [{ + $lhs `,` $rhs (`overflow` `` $overflowFlags^)? + custom(attr-dict) `:` type($res) + }]; + string llvmBuilder = + "$res = builder.Create" # instName # + "($lhs, $rhs, /*Name=*/\"\", op.hasNoUnsignedWrap(), op.hasNoSignedWrap());"; +} class LLVM_FloatArithmeticOp traits = []> : LLVM_ArithmeticOpBase; -def LLVM_SubOp : LLVM_IntArithmeticOp<"sub", "Sub">; -def LLVM_MulOp : LLVM_IntArithmeticOp<"mul", "Mul", [Commutative]>; +def LLVM_AddOp : LLVM_IntArithmeticOpWithOverflowFlag<"add", "Add", + [Commutative]>; +def LLVM_SubOp : LLVM_IntArithmeticOpWithOverflowFlag<"sub", "Sub", []>; +def LLVM_MulOp : LLVM_IntArithmeticOpWithOverflowFlag<"mul", "Mul", + [Commutative]>; def LLVM_UDivOp : LLVM_IntArithmeticOp<"udiv", "UDiv">; def LLVM_SDivOp : LLVM_IntArithmeticOp<"sdiv", "SDiv">; def LLVM_URemOp : LLVM_IntArithmeticOp<"urem", "URem">; @@ -102,7 +124,7 @@ def LLVM_OrOp : LLVM_IntArithmeticOp<"or", "Or"> { let hasFolder = 1; } def LLVM_XOrOp : LLVM_IntArithmeticOp<"xor", "Xor">; -def LLVM_ShlOp : LLVM_IntArithmeticOp<"shl", "Shl"> { +def LLVM_ShlOp : LLVM_IntArithmeticOpWithOverflowFlag<"shl", "Shl", []> { let hasFolder = 1; } def LLVM_LShrOp : LLVM_IntArithmeticOp<"lshr", "LShr">; @@ -262,10 +284,10 @@ def LLVM_GEPOp : LLVM_Op<"getelementptr", [Pure, }]; let builders = [ - OpBuilder<(ins "Type":$resultType, "Type":$basePtrType, "Value":$basePtr, + OpBuilder<(ins "Type":$resultType, "Type":$elementType, "Value":$basePtr, "ValueRange":$indices, CArg<"bool", "false">:$inbounds, CArg<"ArrayRef", "{}">:$attributes)>, - OpBuilder<(ins "Type":$resultType, "Type":$basePtrType, "Value":$basePtr, + OpBuilder<(ins "Type":$resultType, "Type":$elementType, "Value":$basePtr, "ArrayRef":$indices, CArg<"bool", "false">:$inbounds, CArg<"ArrayRef", "{}">:$attributes)>, ]; diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td index de65f3176c46ac8fa0e8362353f57be787969b2a..77ed9db5e71bd150afdc004736cc1306aa2afede 100644 --- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td +++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td @@ -574,6 +574,11 @@ def MatchOp : Op:$ops, OptionalAttr:$interface, OptionalAttr:$op_attrs, - OptionalAttr:$filter_result_type); + OptionalAttr:$filter_result_type, + OptionalAttr:$filter_operand_types); // TODO: variadic results when needed. let results = (outs TransformHandleTypeInterface:$results); @@ -609,6 +615,7 @@ def MatchOp : Op replacements; + static llvm::SmallDenseSet + getPreservedProducerResults(GenericOp producer, GenericOp consumer); }; FailureOr fuseElementwiseOps(RewriterBase &rewriter, OpOperand *fusedOperand); diff --git a/mlir/include/mlir/Dialect/Math/IR/MathOps.td b/mlir/include/mlir/Dialect/Math/IR/MathOps.td index f8e9fd601304b9d1bf985bef81520a1f894ffd53..9742d3d936dff5edd98d8660b2f1b80a08aca7eb 100644 --- a/mlir/include/mlir/Dialect/Math/IR/MathOps.td +++ b/mlir/include/mlir/Dialect/Math/IR/MathOps.td @@ -300,6 +300,35 @@ def Math_CosOp : Math_FloatUnaryOp<"cos"> { let hasFolder = 1; } +//===----------------------------------------------------------------------===// +// AcosOp +//===----------------------------------------------------------------------===// + +def Math_AcosOp : Math_FloatUnaryOp<"acos"> { + let summary = "arcus cosine of the specified value"; + let description = [{ + Syntax: + + ``` + operation ::= ssa-id `=` `math.acos` ssa-use `:` type + ``` + + The `acos` operation computes the arcus cosine of a given value. It takes one + operand of floating point type (i.e., scalar, tensor or vector) and returns one + result of the same type. It has no standard attributes. + + Example: + + ```mlir + // Scalar arcus cosine value. + %a = math.acos %b : f64 + ``` + }]; + let hasFolder = 1; +} + + + //===----------------------------------------------------------------------===// // SinOp //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td index 361e67fd1e19ac6a45e3e92656c8a06b332d6b9a..e6cdba949b1721e9e184fa1fc255d5d93dad3f0c 100644 --- a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td +++ b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td @@ -40,26 +40,27 @@ def Mesh_ClusterOp : Mesh_Op<"cluster", [Symbol]> { determine the layout and the addressing space of the computation distributed across the mesh. - 3. `dim_sizes`: This attribute represents the device assignment along the - axes of the cluster. Each integer in the array corresponds to the number of - devices along a specific axis. If an integer value is 0, it implies that the - number of devices along that axis is unknown. This flexibility allows for - dynamic device assignment or configurations where the exact number of - devices might not be determined during compile time. + 3. `dim_sizes`: This attribute represents the shape of the device cluster. + It uses the same notation as a tensor shape. Also allowing for dynamic + dimensions. + This flexibility allows for dynamic device assignment or configurations + where the exact number of devices might not be determined during compile + time. + For example `2x?x4`. Example: ``` // A device mesh cluster with 3 axes, the total device number is 4 * 8 * 12 // The dimension sizes are 4, 8, 12 - mesh.cluster @mesh0(rank = 3, dim_sizes = [4, 8, 12]) + mesh.cluster @mesh0(rank = 3, dim_sizes = 4x8x12) // A device mesh cluster with 2 axes, the total device number is unknown // The first dimension size is 4 and the second is unknown - mesh.cluster @mesh1(rank = 2, dim_sizes = [4]) + mesh.cluster @mesh1(rank = 2, dim_sizes = 4) // A device mesh cluster with 2 axes, the total device number is unknown // The first dimension size is unknown and the second is 4 - mesh.cluster @mesh2(rank = 2, dim_sizes = [0, 4]) + mesh.cluster @mesh2(rank = 2, dim_sizes = ?x4) // A device mesh cluster with 2 axes, the number of devices along both axes // is unknown @@ -76,7 +77,7 @@ def Mesh_ClusterOp : Mesh_Op<"cluster", [Symbol]> { DefaultValuedAttr:$dim_sizes ); let assemblyFormat = [{ - $sym_name `(` `rank` `=` $rank (`,` `dim_sizes` `=` $dim_sizes^)? `)` + $sym_name `(` `rank` `=` $rank (`,` `dim_sizes` `=` custom($dim_sizes)^)? `)` attr-dict }]; let extraClassDeclaration = [{ @@ -88,7 +89,7 @@ def Mesh_ClusterOp : Mesh_Op<"cluster", [Symbol]> { template void canonicalDimSizes(OutIt outIt) { std::copy(getDimSizes().begin(), getDimSizes().end(), outIt); - std::fill_n(outIt, getRank() - getDimSizes().size(), 0); + std::fill_n(outIt, getRank() - getDimSizes().size(), ::mlir::ShapedType::kDynamic); } }]; let hasVerifier = 1; @@ -210,7 +211,7 @@ def Mesh_AllGatherOp : Mesh_CollectiveCommunicationOpBase<"all_gather", [ Example: ```mlir - mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 2]) + mesh.cluster @mesh0(rank = 2, dim_sizes = 2x2) ... %1 = mesh.all_gather %0 on @mesh0 mesh_axes = [1] gather_axis = 1 : tensor<2x2xi8> -> tensor<2x4xi8> @@ -295,7 +296,7 @@ def Mesh_AllToAllOp : Mesh_CollectiveCommunicationOpBase<"all_to_all", [ Example: ``` - mesh.cluster @mesh0(rank = 1, dim_sizes = [3]) + mesh.cluster @mesh0(rank = 1, dim_sizes = 3) ... %1 = mesh.all_to_all %0 on @mesh0 mesh_axes = [0] split_axis = 0 concat_axis = 0 @@ -527,7 +528,7 @@ def Mesh_ReduceScatterOp : Mesh_CollectiveCommunicationOpBase<"reduce_scatter", across the device group. Example: ``` - mesh.cluster @mesh0(rank = 1, dim_sizes = [2, 2]) + mesh.cluster @mesh0(rank = 1, dim_sizes = 2x2) ... %1 = mesh.reduce_scatter %0 on @mesh0 mesh_axes = [1] reduction = scatter_axis = 0 diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td index 391e77e0c4081a3f843e067c0b90325f64b119ac..9d48b1f1c3f9afa15a76ad768d704146d39c9abc 100644 --- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td +++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td @@ -156,7 +156,7 @@ def DeclareActionAttr : OpenACC_Attr<"DeclareAction", "declare_action"> { } // Device type enumeration. -def OpenACC_DeviceTypeStar : I32EnumAttrCase<"Star", 0, "*">; +def OpenACC_DeviceTypeStar : I32EnumAttrCase<"Star", 0, "star">; def OpenACC_DeviceTypeDefault : I32EnumAttrCase<"Default", 1, "default">; def OpenACC_DeviceTypeHost : I32EnumAttrCase<"Host", 2, "host">; def OpenACC_DeviceTypeMulticore : I32EnumAttrCase<"Multicore", 3, "multicore">; diff --git a/mlir/include/mlir/Dialect/SCF/Transforms/Transforms.h b/mlir/include/mlir/Dialect/SCF/Transforms/Transforms.h index 347beb9e4c64f8cf0660781a3cf91e748795f422..cad51735994538c7da6f1fa22cea8199e0f7efeb 100644 --- a/mlir/include/mlir/Dialect/SCF/Transforms/Transforms.h +++ b/mlir/include/mlir/Dialect/SCF/Transforms/Transforms.h @@ -128,6 +128,13 @@ struct PipeliningOption { /// lambda to generate the predicated version of operations. bool peelEpilogue = true; + /// Control whether the transformation checks that the number of iterations is + /// greater or equal to the number of stages and skip the transformation if + /// this is not the case. If the loop is dynamic and this is set to true and + /// the loop bounds are not static the pipeliner will have to predicate + /// operations in the the prologue/epilogue. + bool supportDynamicLoops = false; + // Callback to predicate operations when the prologue or epilogue are not // peeled. This takes the original operation, an i1 predicate value and the // pattern rewriter. It is expected to replace the given operation with diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBitOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBitOps.td index 91b69576cd1d3177a1128faff94b12e87c599a23..b460c8e68aa0c64219c6c12240b34b26daa8d669 100644 --- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBitOps.td +++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBitOps.td @@ -334,6 +334,8 @@ def SPIRV_BitwiseXorOp : SPIRV_BitBinaryOp<"BitwiseXor", %2 = spirv.BitwiseXor %0, %1 : vector<4xi32> ``` }]; + + let hasFolder = 1; } // ----- @@ -368,6 +370,8 @@ def SPIRV_ShiftLeftLogicalOp : SPIRV_ShiftOp<"ShiftLeftLogical", %5 = spirv.ShiftLeftLogical %3, %4 : vector<3xi32>, vector<3xi16> ``` }]; + + let hasFolder = 1; } // ----- @@ -399,6 +403,8 @@ def SPIRV_ShiftRightArithmeticOp : SPIRV_ShiftOp<"ShiftRightArithmetic", %5 = spirv.ShiftRightArithmetic %3, %4 : vector<3xi32>, vector<3xi16> ``` }]; + + let hasFolder = 1; } // ----- @@ -431,6 +437,8 @@ def SPIRV_ShiftRightLogicalOp : SPIRV_ShiftOp<"ShiftRightLogical", %5 = spirv.ShiftRightLogical %3, %4 : vector<3xi32>, vector<3xi16> ``` }]; + + let hasFolder = 1; } // ----- diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h b/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h index 41ffc929946027ec051d24be468455b941bf1bc4..08d3bb157a0e396105b2117d126ca4bca2588ff1 100644 --- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h +++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h @@ -59,16 +59,6 @@ struct VectorTransformsOptions { vectorTransferSplit = opt; return *this; } - - /// Option to control if vector.transpose can lower to a vector.shape_cast. - /// TODO: ATM it's not possible to lower `vector.shape_cast` to SPIR-V - /// and hence the need for this opt-out. Once the missing support has been - /// added, this option can be removed. - bool useShapeCast = true; - VectorTransformsOptions &setUseShapeCast(bool opt = true) { - useShapeCast = opt; - return *this; - } }; //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/IR/BuiltinAttributes.td b/mlir/include/mlir/IR/BuiltinAttributes.td index be9cbec768d7b23798025dc18c39cf16bf82d255..d9295936ee97bd4ef0c174942c5b2210501d3dcd 100644 --- a/mlir/include/mlir/IR/BuiltinAttributes.td +++ b/mlir/include/mlir/IR/BuiltinAttributes.td @@ -454,9 +454,15 @@ def Builtin_DenseResourceElementsAttr : Builtin_Attr<"DenseResourceElements", Examples: ```mlir - // A tensor referencing a builtin dialect resource, `resource_1`, with two - // unsigned i32 elements. - dense_resource : tensor<2xui32> + "example.user_op"() {attr = dense_resource : tensor<3xi64> } : () -> () + + {-# + dialect_resources: { + builtin: { + blob1: "0x08000000010000000000000002000000000000000300000000000000" + } + } + #-} ``` }]; let parameters = (ins diff --git a/mlir/include/mlir/IR/BuiltinLocationAttributes.td b/mlir/include/mlir/IR/BuiltinLocationAttributes.td index e1656f268795d013a13642d46ab02a4a683ab1ab..dfcc180071f72a8bb28c970f55545033f8749695 100644 --- a/mlir/include/mlir/IR/BuiltinLocationAttributes.td +++ b/mlir/include/mlir/IR/BuiltinLocationAttributes.td @@ -112,8 +112,8 @@ def FusedLoc : Builtin_LocationAttr<"FusedLoc"> { Syntax: ``` - fused-location ::= `fused` fusion-metadata? `[` location (location `,`)* `]` fusion-metadata ::= `<` attribute-value `>` + fused-location ::= `fused` fusion-metadata? `[` (location (`,` location)* )? `]` ``` An instance of a `fused` location represents a grouping of several other @@ -126,7 +126,7 @@ def FusedLoc : Builtin_LocationAttr<"FusedLoc"> { Example: ```mlir - loc(fused["mysource.cc":10:8, "mysource.cc":22:8) + loc(fused["mysource.cc":10:8, "mysource.cc":22:8]) loc(fused<"CSE">["mysource.cc":10:8, "mysource.cc":22:8]) ``` }]; @@ -159,7 +159,7 @@ def NameLoc : Builtin_LocationAttr<"NameLoc"> { This can be useful for representing the locations of variable, or node, definitions. - Example: + #### Example: ```mlir loc("CSE"("mysource.cc":10:8)) @@ -191,6 +191,13 @@ def OpaqueLoc : Builtin_LocationAttr<"OpaqueLoc"> { structure that is external to MLIR and an optional location that can be used if the first one is not suitable. Since it contains an external structure, only the optional location is used during serialization. + + #### Example: + + ```mlir + %0 = "example.operation"() : () -> i32 loc("mysource") + %1 = arith.constant 4 : index loc(callsite("mysum" at "mysource.cc":10:8)) + ``` }]; let parameters = (ins "uintptr_t":$underlyingLocation, "TypeID":$underlyingTypeID, diff --git a/mlir/include/mlir/IR/BuiltinTypes.td b/mlir/include/mlir/IR/BuiltinTypes.td index 1d7772810ae6e8f1b1c89ee585084e171bc8c38f..4cade83dd3c32a1085b52435cfe09cd32b95b2c0 100644 --- a/mlir/include/mlir/IR/BuiltinTypes.td +++ b/mlir/include/mlir/IR/BuiltinTypes.td @@ -47,7 +47,7 @@ def Builtin_Complex : Builtin_Type<"Complex", "complex"> { element type, which is composed of a real and imaginary value of that element type. The element must be a floating point or integer scalar type. - Examples: + #### Example: ```mlir complex @@ -251,6 +251,16 @@ def Builtin_Function : Builtin_Type<"Function", "function"> { The function type can be thought of as a function signature. It consists of a list of formal parameter types and a list of formal result types. + + #### Example: + + ```mlir + func.func @add_one(%arg0 : i64) -> i64 { + %c1 = arith.constant 1 : i64 + %0 = arith.addi %arg0, %c1 : i64 + return %0 : i64 + } + ``` }]; let parameters = (ins "ArrayRef":$inputs, "ArrayRef":$results); let builders = [ @@ -392,10 +402,10 @@ def Builtin_MemRef : Builtin_Type<"MemRef", "memref", [ Syntax: ``` - memref-type ::= `memref` `<` dimension-list-ranked type - (`,` layout-specification)? (`,` memory-space)? `>` layout-specification ::= attribute-value memory-space ::= attribute-value + memref-type ::= `memref` `<` dimension-list-ranked type + (`,` layout-specification)? (`,` memory-space)? `>` ``` A `memref` type is a reference to a region of memory (similar to a buffer @@ -667,8 +677,23 @@ def Builtin_MemRef : Builtin_Type<"MemRef", "memref", [ def Builtin_None : Builtin_Type<"None", "none"> { let summary = "A unit type"; let description = [{ + Syntax: + + ``` + none-type ::= `none` + ``` + NoneType is a unit type, i.e. a type with exactly one possible value, where its value does not have a defined dynamic representation. + + #### Example: + + ```mlir + func.func @none_type() { + %none_val = "foo.unknown_op"() : () -> none + return + } + ``` }]; let extraClassDeclaration = [{ static NoneType get(MLIRContext *context); @@ -692,7 +717,7 @@ def Builtin_Opaque : Builtin_Type<"Opaque", "opaque"> { represented in their raw string form, and can only usefully be tested for type equality. - Examples: + #### Example: ```mlir opaque<"llvm", "struct<(i32, float)>"> @@ -761,7 +786,7 @@ def Builtin_RankedTensor : Builtin_Type<"RankedTensor", "tensor", [ zero sizes are not allowed in some other types, such tensors should be optimized away before lowering tensors to vectors. - Examples: + #### Example: ```mlir // Known rank but unknown dimensions. @@ -846,7 +871,7 @@ def Builtin_Tuple : Builtin_Type<"Tuple", "tuple"> { provides no standard operations for operating on `tuple` types ([rationale](../../Rationale/Rationale/#tuple-types)). - Examples: + #### Example: ```mlir // Empty tuple. @@ -917,7 +942,7 @@ def Builtin_UnrankedMemRef : Builtin_Type<"UnrankedMemRef", "unranked_memref", [ See [MemRefType](#memreftype) for more information on memref types. - Examples: + #### Examples: ```mlir memref<*f32> @@ -990,7 +1015,7 @@ def Builtin_UnrankedTensor : Builtin_Type<"UnrankedTensor", "unranked_tensor", [ unknown rank. See [RankedTensorType](#rankedtensortype) for more information on tensor types. - Examples: + #### Examples: ```mlir tensor<*xf32> @@ -1051,7 +1076,7 @@ def Builtin_Vector : Builtin_Type<"Vector", "vector", [ShapedTypeInterface], "Ty declarations, `vector<0x42xi32>` is invalid because it is interpreted as a 2D vector with shape `(0, 42)` and zero shapes are not allowed. - Examples: + #### Examples: ```mlir // A 2D fixed-length vector of 3x42 i32 elements. diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h index f1fabf95a68b7ad4ac25adf20feba9a564457c7e..86ed14e7ca8439ea235005193c85d1350ccaa307 100644 --- a/mlir/include/mlir/IR/OpImplementation.h +++ b/mlir/include/mlir/IR/OpImplementation.h @@ -226,6 +226,8 @@ public: printArrowTypeList(results); } + void printDimensionList(ArrayRef shape); + /// Class used to automatically end a cyclic region on destruction. class CyclicPrintReset { public: @@ -1762,6 +1764,17 @@ public: const SetVector &referencedResources, AsmResourceBuilder &builder) const {} }; + +//===--------------------------------------------------------------------===// +// Custom printers and parsers. +//===--------------------------------------------------------------------===// + +// Handles custom(...) in TableGen. +void printDimensionList(OpAsmPrinter &printer, Operation *op, + ArrayRef dimensions); +ParseResult parseDimensionList(OpAsmParser &parser, + DenseI64ArrayAttr &dimensions); + } // namespace mlir //===--------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Pass/Pass.h b/mlir/include/mlir/Pass/Pass.h index 5a4df4324ecd1e6d98f97f60d927683315bdcdbe..121b253eb83fea8a3a2cf8f31dab482bdca7671e 100644 --- a/mlir/include/mlir/Pass/Pass.h +++ b/mlir/include/mlir/Pass/Pass.h @@ -9,6 +9,7 @@ #ifndef MLIR_PASS_PASS_H #define MLIR_PASS_PASS_H +#include "mlir/IR/Action.h" #include "mlir/Pass/AnalysisManager.h" #include "mlir/Pass/PassRegistry.h" #include "mlir/Support/LogicalResult.h" @@ -457,6 +458,52 @@ protected: } }; +/// This class encapsulates the "action" of executing a single pass. This allows +/// a user of the Action infrastructure to query information about an action in +/// (for example) a breakpoint context. You could use it like this: +/// +/// auto onBreakpoint = [&](const ActionActiveStack *backtrace) { +/// if (auto passExec = dyn_cast(anAction)) +/// record(passExec.getPass()); +/// return ExecutionContext::Apply; +/// }; +/// ExecutionContext exeCtx(onBreakpoint); +/// +class PassExecutionAction : public tracing::ActionImpl { + using Base = tracing::ActionImpl; + +public: + /// Define a TypeID for this PassExecutionAction. + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(PassExecutionAction) + /// Construct a PassExecutionAction. This is called by the OpToOpPassAdaptor + /// when it calls `executeAction`. + PassExecutionAction(ArrayRef irUnits, const Pass &pass); + + /// The tag required by ActionImpl to identify this action. + static constexpr StringLiteral tag = "pass-execution"; + + /// Print a textual version of this action to `os`. + void print(raw_ostream &os) const override; + + /// Get the pass that will be executed by this action. This is not a class of + /// passes, or all instances of a pass kind, this is a single pass. + const Pass &getPass() const { return pass; } + + /// Get the operation that is the base of this pass. For example, an + /// OperationPass would return a ModuleOp. + Operation *getOp() const; + +public: + /// Reference to the pass being run. Notice that this will *not* extend the + /// lifetime of the pass, and so this class is therefore unsafe to keep past + /// the lifetime of the `executeAction` call. + const Pass &pass; + + /// The base op for this pass. For an OperationPass, we would have a + /// ModuleOp here. + Operation *op; +}; + } // namespace mlir #endif // MLIR_PASS_PASS_H diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h index b8e449dc11df155e2fc5d8d22f6e7814f364839f..b49d2f539453e6a4488ed33f9a759cba8619ebd1 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h @@ -172,6 +172,12 @@ public: /// attributes of LLVMFuncOp `funcOp`. void processFunctionAttributes(llvm::Function *func, LLVMFuncOp funcOp); + /// Sets the integer overflow flags (nsw/nuw) attribute for the imported + /// operation `op` given the original instruction `inst`. Asserts if the + /// operation does not implement the integer overflow flag interface. + void setIntegerOverflowFlagsAttr(llvm::Instruction *inst, + Operation *op) const; + /// Sets the fastmath flags attribute for the imported operation `op` given /// the original instruction `inst`. Asserts if the operation does not /// implement the fastmath interface. diff --git a/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp b/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp index 39b3a0996396ffb2adf31c801a6a72f83ddb9bda..a82c30717e275beaa320999de75afc90862e741d 100644 --- a/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp +++ b/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp @@ -180,7 +180,7 @@ void IntegerRangeAnalysis::visitNonControlFlowArguments( } else if (auto value = llvm::dyn_cast_if_present(*loopBound)) { const IntegerValueRangeLattice *lattice = getLatticeElementFor(op, value); - if (lattice != nullptr) + if (lattice != nullptr && !lattice->getValue().isUninitialized()) return getUpper ? lattice->getValue().getValue().smax() : lattice->getValue().getValue().smin(); } diff --git a/mlir/lib/Bindings/Python/ExecutionEngineModule.cpp b/mlir/lib/Bindings/Python/ExecutionEngineModule.cpp index 3f834259622f7ad627b405a8a36b98761f5c4235..b3df30583fc96315210e58a4aa1fe4eeb8d56780 100644 --- a/mlir/lib/Bindings/Python/ExecutionEngineModule.cpp +++ b/mlir/lib/Bindings/Python/ExecutionEngineModule.cpp @@ -20,7 +20,7 @@ class PyExecutionEngine { public: PyExecutionEngine(MlirExecutionEngine executionEngine) : executionEngine(executionEngine) {} - PyExecutionEngine(PyExecutionEngine &&other) + PyExecutionEngine(PyExecutionEngine &&other) noexcept : executionEngine(other.executionEngine) { other.executionEngine.ptr = nullptr; } diff --git a/mlir/lib/Bindings/Python/IRInterfaces.cpp b/mlir/lib/Bindings/Python/IRInterfaces.cpp index c3aac0b092bc79738f425ef97fbfd1b036a025b7..54cfa56066eb8b96dbc929c8160fd0975e0bf397 100644 --- a/mlir/lib/Bindings/Python/IRInterfaces.cpp +++ b/mlir/lib/Bindings/Python/IRInterfaces.cpp @@ -326,7 +326,7 @@ public: : shape(std::move(shape)), elementType(elementType), attribute(attribute), ranked(true) {} PyShapedTypeComponents(PyShapedTypeComponents &) = delete; - PyShapedTypeComponents(PyShapedTypeComponents &&other) + PyShapedTypeComponents(PyShapedTypeComponents &&other) noexcept : shape(other.shape), elementType(other.elementType), attribute(other.attribute), ranked(other.ranked) {} diff --git a/mlir/lib/Bindings/Python/IRModule.h b/mlir/lib/Bindings/Python/IRModule.h index d99b87d19bbea3cd56ce9cca1d6ba1f377e19ce9..79b7e0c96188c1cc0e4355ef7c1340a9b85da633 100644 --- a/mlir/lib/Bindings/Python/IRModule.h +++ b/mlir/lib/Bindings/Python/IRModule.h @@ -4,6 +4,7 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception //===----------------------------------------------------------------------===// #ifndef MLIR_BINDINGS_PYTHON_IRMODULES_H @@ -53,7 +54,7 @@ public: "cannot construct PyObjectRef with null referrent"); assert(this->object && "cannot construct PyObjectRef with null object"); } - PyObjectRef(PyObjectRef &&other) + PyObjectRef(PyObjectRef &&other) noexcept : referrent(other.referrent), object(std::move(other.object)) { other.referrent = nullptr; assert(!other.object); @@ -484,7 +485,8 @@ public: mlirDialectRegistryDestroy(registry); } PyDialectRegistry(PyDialectRegistry &) = delete; - PyDialectRegistry(PyDialectRegistry &&other) : registry(other.registry) { + PyDialectRegistry(PyDialectRegistry &&other) noexcept + : registry(other.registry) { other.registry = {nullptr}; } diff --git a/mlir/lib/Bindings/Python/Pass.cpp b/mlir/lib/Bindings/Python/Pass.cpp index 588a8e25414c657a7fc492ec3f3370bc98dc6cc7..a68421b61641f69b48da4c82144f330ecd3cbea4 100644 --- a/mlir/lib/Bindings/Python/Pass.cpp +++ b/mlir/lib/Bindings/Python/Pass.cpp @@ -23,7 +23,8 @@ namespace { class PyPassManager { public: PyPassManager(MlirPassManager passManager) : passManager(passManager) {} - PyPassManager(PyPassManager &&other) : passManager(other.passManager) { + PyPassManager(PyPassManager &&other) noexcept + : passManager(other.passManager) { other.passManager.ptr = nullptr; } ~PyPassManager() { diff --git a/mlir/lib/CAPI/IR/IR.cpp b/mlir/lib/CAPI/IR/IR.cpp index d1ee1b774c3447872ad0e53e2a409a68de234923..ac9889df11f80d505396b3628a5814286f0fb47a 100644 --- a/mlir/lib/CAPI/IR/IR.cpp +++ b/mlir/lib/CAPI/IR/IR.cpp @@ -986,6 +986,10 @@ MlirOperation mlirOpOperandGetOwner(MlirOpOperand opOperand) { return wrap(unwrap(opOperand)->getOwner()); } +MlirValue mlirOpOperandGetValue(MlirOpOperand opOperand) { + return wrap(unwrap(opOperand)->get()); +} + unsigned mlirOpOperandGetOperandNumber(MlirOpOperand opOperand) { return unwrap(opOperand)->getOperandNumber(); } diff --git a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp index c3c9780318a9e9b8b62648ec2f140517a6b49172..c9d7c0c313b5c8d2d6fe74406dc8ecb5c3272672 100644 --- a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp +++ b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp @@ -484,12 +484,6 @@ struct ConvertArmSMEToSCFPass target.addLegalDialect(); target.addIllegalOp(); - target.addDynamicallyLegalOp([](vector::PrintOp op) { - if (!op.getSource()) - return true; - VectorType vectorType = dyn_cast(op.getPrintType()); - return !vectorType || !arm_sme::isValidSMETileVectorType(vectorType); - }); if (failed(applyPartialConversion(getOperation(), target, std::move(patterns)))) signalPassFailure(); diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp index 75dee09d2f64fd04995cee09aadce255b39353b7..b68baffb5a8565c884a13b62ed799f352d7f9487 100644 --- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp +++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp @@ -1340,18 +1340,18 @@ LogicalResult ConvertSetDefaultDeviceOpToGpuRuntimeCallPattern::matchAndRewrite( } template -static Value genConstInt32From(OpBuilder &builder, Location loc, T TValue) { +static Value genConstInt32From(OpBuilder &builder, Location loc, T tValue) { Type llvmInt32Type = builder.getIntegerType(32); return builder.create(loc, llvmInt32Type, - static_cast(TValue)); + static_cast(tValue)); } template -static Value genConstFloat32From(OpBuilder &builder, Location loc, T TValue) { +static Value genConstFloat32From(OpBuilder &builder, Location loc, T tValue) { Type llvmFloat32Type = builder.getF32Type(); return builder.create( loc, llvmFloat32Type, - builder.getF32FloatAttr(static_cast(TValue))); + builder.getF32FloatAttr(static_cast(tValue))); } LogicalResult ConvertCreateDnTensorOpToGpuRuntimeCallPattern::matchAndRewrite( @@ -1629,7 +1629,7 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite( auto stream = adaptor.getAsyncDependencies().front(); Value bufferSize; if (is2To4Sparsity(op.getSpmatA())) { - auto prune_flag = + auto pruneFlag = genConstInt32From(rewriter, loc, get2To4PruneFlag(op.getSpmatA())); auto computeType = genConstInt32From( rewriter, loc, getCuSparseLtDataTypeFrom(adaptor.getComputeType())); @@ -1641,7 +1641,7 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite( .create(loc, rewriter, {bufferSize, modeA, modeB, adaptor.getSpmatA(), adaptor.getDnmatB(), adaptor.getDnmatC(), computeType, - prune_flag, stream}) + pruneFlag, stream}) .getResult(); auto bufferSizePtr1 = rewriter.create( diff --git a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp index 103c1fb8c3822ec91ce1367a36ba84636b93d02a..27c2cb935207142ccac25cac25d2df3f4ee8cbfc 100644 --- a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp +++ b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp @@ -162,6 +162,7 @@ ScalarOpToLibmCall::matchAndRewrite(Op op, void mlir::populateMathToLibmConversionPatterns(RewritePatternSet &patterns) { MLIRContext *ctx = patterns.getContext(); + populatePatternsForOp(patterns, ctx, "acosf", "acos"); populatePatternsForOp(patterns, ctx, "atan2f", "atan2"); populatePatternsForOp(patterns, ctx, "atanf", "atan"); populatePatternsForOp(patterns, ctx, "cbrtf", "cbrt"); diff --git a/mlir/lib/Conversion/MemRefToSPIRV/MapMemRefStorageClassPass.cpp b/mlir/lib/Conversion/MemRefToSPIRV/MapMemRefStorageClassPass.cpp index 35a3af07940a83051253a13b7ae0cf00d2616a04..c6ef5be2494ad3cf84ebb9a04da50d1628427b67 100644 --- a/mlir/lib/Conversion/MemRefToSPIRV/MapMemRefStorageClassPass.cpp +++ b/mlir/lib/Conversion/MemRefToSPIRV/MapMemRefStorageClassPass.cpp @@ -205,7 +205,7 @@ spirv::MemorySpaceToStorageClassConverter::MemorySpaceToStorageClassConverter( static bool isLegalType(Type type) { if (auto memRefType = dyn_cast(type)) { Attribute spaceAttr = memRefType.getMemorySpace(); - return spaceAttr && isa(spaceAttr); + return isa_and_nonnull(spaceAttr); } return true; } diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp index baab40020ac2e9e16fdaacc4ea5f3d332ce2b8b8..a9c3b0a71ef0d7e55d999ad470ca577f1f479f67 100644 --- a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp +++ b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp @@ -856,7 +856,7 @@ static void foldSwitchToBool(std::unique_ptr &node) { // If the node only contains one child, collapse it into a boolean predicate // node. if (children.size() == 1) { - auto childIt = children.begin(); + auto *childIt = children.begin(); node = std::make_unique( node->getPosition(), node->getQuestion(), childIt->first, std::move(childIt->second), std::move(node->getFailureNode())); diff --git a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp index 429d1137b6f3781d35badce67744eda87585822c..56fcf25c9ddcc3cafd0ca5e6d4c0ea6eeb2f1436 100644 --- a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp +++ b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp @@ -303,8 +303,9 @@ static bool supportsMMaMatrixType(Operation *op, bool useNvGpu) { /// `getSlice`. In scf.for we only want to include as part of the slice elements /// that are part of the use/def chain. static SetVector -getSliceContract(Operation *op, BackwardSliceOptions backwardSliceOptions, - ForwardSliceOptions forwardSliceOptions) { +getSliceContract(Operation *op, + const BackwardSliceOptions &backwardSliceOptions, + const ForwardSliceOptions &forwardSliceOptions) { SetVector slice; slice.insert(op); unsigned currentIndex = 0; @@ -455,7 +456,8 @@ struct CombineTransferReadOpTranspose final Type resultType = op.getType(); Operation *extOp; if ((extOp = source.getDefiningOp()) || - (extOp = source.getDefiningOp())) { + (extOp = source.getDefiningOp()) || + (extOp = source.getDefiningOp())) { source = extOp->getOperand(0); resultType = VectorType::get(cast(resultType).getShape(), @@ -493,9 +495,12 @@ struct CombineTransferReadOpTranspose final if (isa(extOp)) result = rewriter.create(loc, op.getType(), result) .getResult(); - else + else if (isa(extOp)) result = rewriter.create(loc, op.getType(), result) .getResult(); + else + result = rewriter.create(loc, op.getType(), result) + .getResult(); } rewriter.replaceOp(op, result); @@ -553,7 +558,7 @@ convertTransferReadOp(RewriterBase &rewriter, vector::TransferReadOp op, auto elType = op.getVectorType().getElementType(); const char *fragType = inferFragType(op); if (op->hasOneUse()) { - auto user = *op->user_begin(); + auto *user = *op->user_begin(); // Infer the signedness of the mma type from the integer extend. bool isSignedExtend = isa(user); if (isSignedExtend || isa(user)) { diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp index a7fc7ddec26e618871cb51ca2437c8a084c01b77..7f2f3c3410c33b4bdc3420343c19178c119cea68 100644 --- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp +++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp @@ -4474,6 +4474,17 @@ LogicalResult AffineVectorStoreOp::verify() { // DelinearizeIndexOp //===----------------------------------------------------------------------===// +LogicalResult AffineDelinearizeIndexOp::inferReturnTypes( + MLIRContext *context, std::optional<::mlir::Location> location, + ValueRange operands, DictionaryAttr attributes, OpaqueProperties properties, + RegionRange regions, SmallVectorImpl &inferredReturnTypes) { + AffineDelinearizeIndexOpAdaptor adaptor(operands, attributes, properties, + regions); + inferredReturnTypes.assign(adaptor.getBasis().size(), + IndexType::get(context)); + return success(); +} + void AffineDelinearizeIndexOp::build(OpBuilder &builder, OperationState &result, Value linearIndex, ArrayRef basis) { diff --git a/mlir/lib/Dialect/Affine/IR/CMakeLists.txt b/mlir/lib/Dialect/Affine/IR/CMakeLists.txt index 9e3c1161fd92a094f2dc6752461275f94066f430..7f7a01be891e05310c48059a7daf8d78e9954ae2 100644 --- a/mlir/lib/Dialect/Affine/IR/CMakeLists.txt +++ b/mlir/lib/Dialect/Affine/IR/CMakeLists.txt @@ -15,6 +15,7 @@ add_mlir_dialect_library(MLIRAffineDialect MLIRArithDialect MLIRDialectUtils MLIRIR + MLIRInferTypeOpInterface MLIRLoopLikeInterface MLIRMemRefDialect MLIRShapedOpInterfaces diff --git a/mlir/lib/Dialect/Affine/Utils/CMakeLists.txt b/mlir/lib/Dialect/Affine/Utils/CMakeLists.txt index c2ce43f4eebb26110f54cb200f0214025299f7bc..ef6e0dbf45d3a9f6ede33c7fb816592cb95d2a54 100644 --- a/mlir/lib/Dialect/Affine/Utils/CMakeLists.txt +++ b/mlir/lib/Dialect/Affine/Utils/CMakeLists.txt @@ -12,6 +12,7 @@ add_mlir_dialect_library(MLIRAffineUtils MLIRAffineAnalysis MLIRAnalysis MLIRArithUtils + MLIRFuncDialect MLIRMemRefDialect MLIRTransformUtils MLIRViewLikeInterface diff --git a/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp b/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp index 6d8706775758e2004ec9eb0054f89a61c49e7a27..5529dccaf1de6fb8dba1cbf5a465fbec066823fb 100644 --- a/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp +++ b/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp @@ -58,10 +58,12 @@ LogicalResult ConstantOp::verify() { } auto complexEltTy = getType().getElementType(); - auto re = llvm::dyn_cast(arrayAttr[0]); - auto im = llvm::dyn_cast(arrayAttr[1]); - if (!re || !im) - return emitOpError("requires attribute's elements to be float attributes"); + if (!isa(arrayAttr[0]) || + !isa(arrayAttr[1])) + return emitOpError( + "requires attribute's elements to be float or integer attributes"); + auto re = llvm::dyn_cast(arrayAttr[0]); + auto im = llvm::dyn_cast(arrayAttr[1]); if (complexEltTy != re.getType() || complexEltTy != im.getType()) { return emitOpError() << "requires attribute's element types (" << re.getType() << ", " diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index c67bba04d697156c62098ded0975042a18a50797..53e1088f620d7eb46aea097113ccd62f55814556 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -69,7 +69,13 @@ static ParseResult parseLLVMOpAttrs(OpAsmParser &parser, static void printLLVMOpAttrs(OpAsmPrinter &printer, Operation *op, DictionaryAttr attrs) { - printer.printOptionalAttrDict(processFMFAttr(attrs.getValue())); + auto filteredAttrs = processFMFAttr(attrs.getValue()); + if (auto iface = dyn_cast(op)) + printer.printOptionalAttrDict( + filteredAttrs, + /*elidedAttrs=*/{iface.getIntegerOverflowAttrName()}); + else + printer.printOptionalAttrDict(filteredAttrs); } /// Verifies `symbol`'s use in `op` to ensure the symbol is a valid and diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp index b094c650ff1932c04301b7c611eda2d7cea27c6b..cf900ac0be8fd27953657e0253eaab22f2a80465 100644 --- a/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp +++ b/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp @@ -161,7 +161,10 @@ static std::optional gepToByteOffset(DataLayout &layout, GEPOp gep) { IntegerAttr indexInt = llvm::dyn_cast_if_present(index); if (!indexInt) return std::nullopt; - indices.push_back(indexInt.getInt()); + int32_t gepIndex = indexInt.getInt(); + if (gepIndex < 0) + return std::nullopt; + indices.push_back(static_cast(gepIndex)); } uint64_t offset = indices[0] * layout.getTypeSize(gep.getElemType()); diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp index e3713457e84123b0b31cd7f7b480b6019fa41f3b..54055aefbc5127c68b470a3ca999e8f6719592a9 100644 --- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp +++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp @@ -1171,6 +1171,7 @@ transform::MatchOp::apply(transform::TransformRewriter &rewriter, } SmallVector res; + bool incorrectNumOperandTypes = false; auto matchFun = [&](Operation *op) { if (getOps().has_value() && !strs.contains(op->getName().getStringRef())) return; @@ -1210,12 +1211,47 @@ transform::MatchOp::apply(transform::TransformRewriter &rewriter, return; } + if (getFilterOperandTypes().has_value()) { + mlir::ArrayAttr types = getFilterOperandTypes().value(); + auto operandTypes = op->getOperandTypes(); + + if (types.size() == 1) { + // All the operands must must be equal to the specified type + auto typeattr = + dyn_cast(getFilterOperandTypes().value()[0]); + Type t = typeattr.getValue().cast<::mlir::Type>(); + if (!llvm::all_of(op->getOperandTypes(), + [&](Type operandType) { return operandType == t; })) + return; + } else { + // The operand types must match all the types in the list (in the same + // order in with they are specified) + if (types.size() != operandTypes.size()) { + incorrectNumOperandTypes = true; + return; + } + + for (auto [attr, operandType] : + llvm::zip_equal(getFilterOperandTypes().value(), operandTypes)) { + auto typeattr = cast(attr); + Type type = typeattr.getValue().cast<::mlir::Type>(); + + if (type != operandType) + return; + } + } + } + // All constraints are satisfied. res.push_back(op); return; }; (*payloadOps.begin())->walk(matchFun); + if (incorrectNumOperandTypes) + return emitDefiniteFailure("If filter_operand_types contains more than a " + "type, then it must contain as much types as " + "the number of operands in the target ops"); results.set(cast(getResult()), res); return DiagnosedSilenceableFailure::success(); } diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp index f0393e44fc00c277de77c6895fb0f9bbdf5be141..3eb91190751ef1347d72a89017ccabe9cc539a24 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp @@ -71,6 +71,25 @@ static AffineMap getIndexingMapOfProducerOperandsInCoordinatesOfFusedOp( return t1.compose(fusedConsumerArgIndexMap); } +/// Returns a set of indices of the producer's results which would +/// be preserved after the fusion. +llvm::SmallDenseSet +ElementwiseOpFusionResult::getPreservedProducerResults(GenericOp producer, + GenericOp consumer) { + llvm::SmallDenseSet preservedProducerResults; + for (const auto &producerResult : llvm::enumerate(producer->getResults())) { + auto *outputOperand = producer.getDpsInitOperand(producerResult.index()); + if (producer.payloadUsesValueFromOperand(outputOperand) || + !producer.canOpOperandsBeDropped(outputOperand) || + llvm::any_of(producerResult.value().getUsers(), [&](Operation *user) { + return user != consumer.getOperation(); + })) { + preservedProducerResults.insert(producerResult.index()); + } + } + return preservedProducerResults; +} + /// Conditions for elementwise fusion of generic operations. bool mlir::linalg::areElementwiseOpsFusable(OpOperand *fusedOperand) { if (!fusedOperand) @@ -285,17 +304,9 @@ mlir::linalg::fuseElementwiseOps(RewriterBase &rewriter, assert(consumer.isDpsInput(fusedOperand) && "expected producer of input operand"); /// Find the results of the producer that have uses outside of the consumer. - llvm::SmallDenseSet preservedProducerResults; - for (const auto &producerResult : llvm::enumerate(producer->getResults())) { - auto *outputOperand = producer.getDpsInitOperand(producerResult.index()); - if (producer.payloadUsesValueFromOperand(outputOperand) || - !producer.canOpOperandsBeDropped(outputOperand) || - llvm::any_of(producerResult.value().getUsers(), [&](Operation *user) { - return user != consumer.getOperation(); - })) { - preservedProducerResults.insert(producerResult.index()); - } - } + llvm::SmallDenseSet preservedProducerResults = + ElementwiseOpFusionResult::getPreservedProducerResults(producer, + consumer); // Compute the fused operands list and indexing maps. SmallVector fusedInputOperands, fusedOutputOperands; @@ -422,11 +433,6 @@ public: if (!controlFn(&opOperand)) continue; - // Find the producer of the operand. - FailureOr fusionResult = - fuseElementwiseOps(rewriter, &opOperand); - if (failed(fusionResult)) - return rewriter.notifyMatchFailure(genericOp, "fusion failed"); Operation *producer = opOperand.get().getDefiningOp(); // Do not fuse a sparse-in/dense-out operation, as the @@ -435,6 +441,12 @@ public: !sparse_tensor::hasAnySparseResult(producer)) return failure(); + // Find the producer of the operand. + FailureOr fusionResult = + fuseElementwiseOps(rewriter, &opOperand); + if (failed(fusionResult)) + return rewriter.notifyMatchFailure(genericOp, "fusion failed"); + // Perform the fusion. for (auto [origVal, replacement] : fusionResult->replacements) { rewriter.replaceUsesWithIf(origVal, replacement, [&](OpOperand &use) { diff --git a/mlir/lib/Dialect/Math/IR/MathOps.cpp b/mlir/lib/Dialect/Math/IR/MathOps.cpp index 28d1c062f235e60b574190f53a41c84c2ef74e14..066a21c76f7d1c67537253a29223d62e67e583f0 100644 --- a/mlir/lib/Dialect/Math/IR/MathOps.cpp +++ b/mlir/lib/Dialect/Math/IR/MathOps.cpp @@ -41,6 +41,24 @@ OpFoldResult math::AbsIOp::fold(FoldAdaptor adaptor) { [](const APInt &a) { return a.abs(); }); } +//===----------------------------------------------------------------------===// +// AcosOp folder +//===----------------------------------------------------------------------===// + +OpFoldResult math::AcosOp::fold(FoldAdaptor adaptor) { + return constFoldUnaryOpConditional( + adaptor.getOperands(), [](const APFloat &a) -> std::optional { + switch (a.getSizeInBits(a.getSemantics())) { + case 64: + return APFloat(acos(a.convertToDouble())); + case 32: + return APFloat(acosf(a.convertToFloat())); + default: + return {}; + } + }); +} + //===----------------------------------------------------------------------===// // AtanOp folder //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp index 3b89860c14d93623c3d42b609de3fc9df65f2714..1ba95f21ec7f3d5c9cd3934386cbcab2cd46875d 100644 --- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp +++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp @@ -58,11 +58,6 @@ static SmallVector &canonicalizeSetAsVector(SmallVector &vec) { return vec; } -template -static bool isMeshDimensionDynamic(DimSize size) { - return size <= DimSize(0); -} - using MeshAxis = int16_t; namespace { @@ -161,9 +156,9 @@ LogicalResult ClusterOp::verify() { "rank of dim_sizes is not expected to be larger than rank of cluster"); for (int64_t dimSize : dimSizes) { - if (dimSize < 0) - return emitOpError( - "dimension size of a mesh cluster is expected to be non-negative"); + if (dimSize < 0 && !ShapedType::isDynamic(dimSize)) + return emitOpError("dimension size of a mesh cluster is expected to be " + "non-negative or dynamic"); } return success(); @@ -316,7 +311,7 @@ static int64_t collectiveDeviceGroupSize(ArrayRef meshAxes, int64_t res = 1; for (MeshAxis axis : meshAxes) { - if (isMeshDimensionDynamic(meshShape[axis])) { + if (ShapedType::isDynamic(meshShape[axis])) { return ShapedType::kDynamic; } assert(size_t(axis) < meshShape.size()); diff --git a/mlir/lib/Dialect/SCF/IR/CMakeLists.txt b/mlir/lib/Dialect/SCF/IR/CMakeLists.txt index 9882b843c285e5a3785577a538f3faf52b3d43a8..423e1c3e1e042c22e98beff1abc7a5a8e9ce948a 100644 --- a/mlir/lib/Dialect/SCF/IR/CMakeLists.txt +++ b/mlir/lib/Dialect/SCF/IR/CMakeLists.txt @@ -11,12 +11,13 @@ add_mlir_dialect_library(MLIRSCFDialect LINK_LIBS PUBLIC MLIRArithDialect - MLIRBufferizationDialect MLIRControlFlowDialect + MLIRDialectUtils MLIRFunctionInterfaces MLIRIR MLIRLoopLikeInterface MLIRSideEffectInterfaces + MLIRTensorDialect MLIRValueBoundsOpInterface ) diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp index 2e7382160f8eb909fc97aa3e5a42e1ccdd561e4b..55bb5788108bdb8cae2df031f9b607253d0c9379 100644 --- a/mlir/lib/Dialect/SCF/IR/SCF.cpp +++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp @@ -9,7 +9,6 @@ #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Arith/Utils/Utils.h" -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/SCF/IR/DeviceMappingInterface.h" @@ -1082,139 +1081,12 @@ struct ForOpTensorCastFolder : public OpRewritePattern { } }; -/// Canonicalize the iter_args of an scf::ForOp that involve a -/// `bufferization.to_tensor` and for which only the last loop iteration is -/// actually visible outside of the loop. The canonicalization looks for a -/// pattern such as: -/// ``` -/// %t0 = ... : tensor_type -/// %0 = scf.for ... iter_args(%bb0 : %t0) -> (tensor_type) { -/// ... -/// // %m is either buffer_cast(%bb00) or defined above the loop -/// %m... : memref_type -/// ... // uses of %m with potential inplace updates -/// %new_tensor = bufferization.to_tensor %m : memref_type -/// ... -/// scf.yield %new_tensor : tensor_type -/// } -/// ``` -/// -/// `%bb0` may have either 0 or 1 use. If it has 1 use it must be exactly a -/// `%m = buffer_cast %bb0` op that feeds into the yielded -/// `bufferization.to_tensor` op. -/// -/// If no aliasing write to the memref `%m`, from which `%new_tensor`is loaded, -/// occurs between `bufferization.to_tensor and yield then the value %0 -/// visible outside of the loop is the last `bufferization.to_tensor` -/// produced in the loop. -/// -/// For now, we approximate the absence of aliasing by only supporting the case -/// when the bufferization.to_tensor is the operation immediately preceding -/// the yield. -// -/// The canonicalization rewrites the pattern as: -/// ``` -/// // %m is either a buffer_cast or defined above -/// %m... : memref_type -/// scf.for ... iter_args(%bb0 : %t0) -> (tensor_type) { -/// ... // uses of %m with potential inplace updates -/// scf.yield %bb0: tensor_type -/// } -/// %0 = bufferization.to_tensor %m : memref_type -/// ``` -/// -/// A later bbArg canonicalization will further rewrite as: -/// ``` -/// // %m is either a buffer_cast or defined above -/// %m... : memref_type -/// scf.for ... { // no iter_args -/// ... // uses of %m with potential inplace updates -/// } -/// %0 = bufferization.to_tensor %m : memref_type -/// ``` -struct LastTensorLoadCanonicalization : public OpRewritePattern { - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(ForOp forOp, - PatternRewriter &rewriter) const override { - assert(std::next(forOp.getRegion().begin()) == forOp.getRegion().end() && - "unexpected multiple blocks"); - - Location loc = forOp.getLoc(); - DenseMap replacements; - for (BlockArgument bbArg : forOp.getRegionIterArgs()) { - unsigned idx = bbArg.getArgNumber() - /*numIv=*/1; - auto yieldOp = - cast(forOp.getRegion().front().getTerminator()); - Value yieldVal = yieldOp->getOperand(idx); - auto tensorLoadOp = yieldVal.getDefiningOp(); - bool isTensor = llvm::isa(bbArg.getType()); - - bufferization::ToMemrefOp tensorToMemref; - // Either bbArg has no use or it has a single buffer_cast use. - if (bbArg.hasOneUse()) - tensorToMemref = - dyn_cast(*bbArg.getUsers().begin()); - if (!isTensor || !tensorLoadOp || (!bbArg.use_empty() && !tensorToMemref)) - continue; - // If tensorToMemref is present, it must feed into the `ToTensorOp`. - if (tensorToMemref && tensorLoadOp.getMemref() != tensorToMemref) - continue; - // TODO: Any aliasing write of tensorLoadOp.memref() nested under `forOp` - // must be before `ToTensorOp` in the block so that the lastWrite - // property is not subject to additional side-effects. - // For now, we only support the case when ToTensorOp appears - // immediately before the terminator. - if (tensorLoadOp->getNextNode() != yieldOp) - continue; - - // Clone the optional tensorToMemref before forOp. - if (tensorToMemref) { - rewriter.setInsertionPoint(forOp); - rewriter.replaceOpWithNewOp( - tensorToMemref, tensorToMemref.getMemref().getType(), - tensorToMemref.getTensor()); - } - - // Clone the tensorLoad after forOp. - rewriter.setInsertionPointAfter(forOp); - Value newTensorLoad = rewriter.create( - loc, tensorLoadOp.getMemref()); - Value forOpResult = forOp.getResult(bbArg.getArgNumber() - /*iv=*/1); - replacements.insert(std::make_pair(forOpResult, newTensorLoad)); - - // Make the terminator just yield the bbArg, the old tensorLoadOp + the - // old bbArg (that is now directly yielded) will canonicalize away. - rewriter.startRootUpdate(yieldOp); - yieldOp.setOperand(idx, bbArg); - rewriter.finalizeRootUpdate(yieldOp); - } - if (replacements.empty()) - return failure(); - - // We want to replace a subset of the results of `forOp`. rewriter.replaceOp - // replaces the whole op and erase it unconditionally. This is wrong for - // `forOp` as it generally contains ops with side effects. - // Instead, use `rewriter.replaceOpWithIf`. - SmallVector newResults; - newResults.reserve(forOp.getNumResults()); - for (Value v : forOp.getResults()) { - auto it = replacements.find(v); - newResults.push_back((it != replacements.end()) ? it->second : v); - } - unsigned idx = 0; - rewriter.replaceOpWithIf(forOp, newResults, [&](OpOperand &op) { - return op.get() != newResults[idx++]; - }); - return success(); - } -}; } // namespace void ForOp::getCanonicalizationPatterns(RewritePatternSet &results, MLIRContext *context) { - results.add(context); + results.add( + context); } std::optional ForOp::getConstantStep() { diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp index 20fa8089201aa19909d3cc50110aa9355e88ddb5..6c36600975a5975a85a989f706e509c2bd558685 100644 --- a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp +++ b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp @@ -44,9 +44,10 @@ protected: unsigned maxStage = 0; DenseMap stages; std::vector opOrder; - int64_t ub; - int64_t lb; - int64_t step; + Value ub; + Value lb; + Value step; + bool dynamicLoop; PipeliningOption::AnnotationlFnType annotateFn = nullptr; bool peelEpilogue; PipeliningOption::PredicateOpFn predicateFn = nullptr; @@ -96,25 +97,41 @@ bool LoopPipelinerInternal::initializeLoopInfo( ForOp op, const PipeliningOption &options) { LDBG("Start initializeLoopInfo"); forOp = op; - auto upperBoundCst = - forOp.getUpperBound().getDefiningOp(); - auto lowerBoundCst = - forOp.getLowerBound().getDefiningOp(); - auto stepCst = forOp.getStep().getDefiningOp(); + ub = forOp.getUpperBound(); + lb = forOp.getLowerBound(); + step = forOp.getStep(); + + dynamicLoop = true; + auto upperBoundCst = getConstantIntValue(ub); + auto lowerBoundCst = getConstantIntValue(lb); + auto stepCst = getConstantIntValue(step); if (!upperBoundCst || !lowerBoundCst || !stepCst) { - LDBG("--no constant bounds or step -> BAIL"); - return false; + if (!options.supportDynamicLoops) { + LDBG("--dynamic loop not supported -> BAIL"); + return false; + } + } else { + int64_t ubImm = upperBoundCst.value(); + int64_t lbImm = lowerBoundCst.value(); + int64_t stepImm = stepCst.value(); + int64_t numIteration = ceilDiv(ubImm - lbImm, stepImm); + if (numIteration > maxStage) { + dynamicLoop = false; + } else if (!options.supportDynamicLoops) { + LDBG("--fewer loop iterations than pipeline stages -> BAIL"); + return false; + } } - ub = upperBoundCst.value(); - lb = lowerBoundCst.value(); - step = stepCst.value(); peelEpilogue = options.peelEpilogue; predicateFn = options.predicateFn; - if (!peelEpilogue && predicateFn == nullptr) { + if ((!peelEpilogue || dynamicLoop) && predicateFn == nullptr) { LDBG("--no epilogue or predicate set -> BAIL"); return false; } - int64_t numIteration = ceilDiv(ub - lb, step); + if (dynamicLoop && peelEpilogue) { + LDBG("--dynamic loop doesn't support epilogue yet -> BAIL"); + return false; + } std::vector> schedule; options.getScheduleFn(forOp, schedule); if (schedule.empty()) { @@ -128,10 +145,6 @@ bool LoopPipelinerInternal::initializeLoopInfo( stages[opSchedule.first] = opSchedule.second; opOrder.push_back(opSchedule.first); } - if (numIteration <= maxStage) { - LDBG("--fewer loop iterations than pipeline stages -> BAIL"); - return false; - } // All operations need to have a stage. for (Operation &op : forOp.getBody()->without_terminator()) { @@ -204,10 +217,31 @@ void LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) { setValueMapping(arg, operand.get(), 0); } auto yield = cast(forOp.getBody()->getTerminator()); + Location loc = forOp.getLoc(); + SmallVector predicates(maxStage); for (int64_t i = 0; i < maxStage; i++) { + if (dynamicLoop) { + Type t = ub.getType(); + // pred = ub > lb + (i * step) + Value iv = rewriter.create( + loc, lb, + rewriter.create( + loc, step, + rewriter.create( + loc, rewriter.getIntegerAttr(t, i)))); + predicates[i] = rewriter.create( + loc, arith::CmpIPredicate::slt, iv, ub); + } + // special handling for induction variable as the increment is implicit. - Value iv = - rewriter.create(forOp.getLoc(), lb + i * step); + // iv = lb + i * step + Type t = lb.getType(); + Value iv = rewriter.create( + loc, lb, + rewriter.create( + loc, step, + rewriter.create(loc, + rewriter.getIntegerAttr(t, i)))); setValueMapping(forOp.getInductionVar(), iv, i); for (Operation *op : opOrder) { if (stages[op] > i) @@ -220,6 +254,12 @@ void LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) { newOperand->set(replacement); } }); + int predicateIdx = i - stages[op]; + if (predicates[predicateIdx]) { + newOp = predicateFn(rewriter, newOp, predicates[predicateIdx]); + assert(newOp && "failed to predicate op."); + } + rewriter.setInsertionPointAfter(newOp); if (annotateFn) annotateFn(newOp, PipeliningOption::PipelinerPart::Prologue, i); for (unsigned destId : llvm::seq(unsigned(0), op->getNumResults())) { @@ -326,9 +366,16 @@ scf::ForOp LoopPipelinerInternal::createKernelLoop( // `numStages - 1` iterations. Then we adjust the upper bound to remove those // iterations. Value newUb = forOp.getUpperBound(); - if (peelEpilogue) - newUb = rewriter.create(forOp.getLoc(), - ub - maxStage * step); + if (peelEpilogue) { + Type t = ub.getType(); + Location loc = forOp.getLoc(); + // newUb = ub - maxStage * step + Value maxStageValue = rewriter.create( + loc, rewriter.getIntegerAttr(t, maxStage)); + Value maxStageByStep = + rewriter.create(loc, step, maxStageValue); + newUb = rewriter.create(loc, ub, maxStageByStep); + } auto newForOp = rewriter.create(forOp.getLoc(), forOp.getLowerBound(), newUb, forOp.getStep(), newLoopArg); @@ -358,9 +405,17 @@ LogicalResult LoopPipelinerInternal::createKernel( SmallVector predicates(maxStage + 1, nullptr); if (!peelEpilogue) { // Create a predicate for each stage except the last stage. + Location loc = newForOp.getLoc(); + Type t = ub.getType(); for (unsigned i = 0; i < maxStage; i++) { - Value c = rewriter.create( - newForOp.getLoc(), ub - (maxStage - i) * step); + // c = ub - (maxStage - i) * step + Value c = rewriter.create( + loc, ub, + rewriter.create( + loc, step, + rewriter.create( + loc, rewriter.getIntegerAttr(t, int64_t(maxStage - i))))); + Value pred = rewriter.create( newForOp.getLoc(), arith::CmpIPredicate::slt, newForOp.getInductionVar(), c); @@ -383,8 +438,14 @@ LogicalResult LoopPipelinerInternal::createKernel( // version incremented based on the stage where it is used. if (operand->get() == forOp.getInductionVar()) { rewriter.setInsertionPoint(newOp); - Value offset = rewriter.create( - forOp.getLoc(), (maxStage - stages[op]) * step); + + // offset = (maxStage - stages[op]) * step + Type t = step.getType(); + Value offset = rewriter.create( + forOp.getLoc(), step, + rewriter.create( + forOp.getLoc(), + rewriter.getIntegerAttr(t, maxStage - stages[op]))); Value iv = rewriter.create( forOp.getLoc(), newForOp.getInductionVar(), offset); nestedNewOp->setOperand(operand->getOperandNumber(), iv); @@ -508,8 +569,24 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter) { // Emit different versions of the induction variable. They will be // removed by dead code if not used. for (int64_t i = 0; i < maxStage; i++) { - Value newlastIter = rewriter.create( - forOp.getLoc(), lb + step * ((((ub - 1) - lb) / step) - i)); + Location loc = forOp.getLoc(); + Type t = lb.getType(); + Value minusOne = + rewriter.create(loc, rewriter.getIntegerAttr(t, -1)); + // number of iterations = ((ub - 1) - lb) / step + Value totalNumIteration = rewriter.create( + loc, + rewriter.create( + loc, rewriter.create(loc, ub, minusOne), lb), + step); + // newLastIter = lb + step * ((((ub - 1) - lb) / step) - i) + Value minusI = + rewriter.create(loc, rewriter.getIntegerAttr(t, -i)); + Value newlastIter = rewriter.create( + loc, lb, + rewriter.create( + loc, step, + rewriter.create(loc, totalNumIteration, minusI))); setValueMapping(forOp.getInductionVar(), newlastIter, maxStage - i); } // Emit `maxStage - 1` epilogue part that includes operations from stages diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp index 22cb9bf718e36f4e58a7fecaa588acf1083341d0..9de1707dfca465bf3bf2297099e6be1bc9547034 100644 --- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp +++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp @@ -709,6 +709,195 @@ OpFoldResult spirv::LogicalOrOp::fold(FoldAdaptor adaptor) { return Attribute(); } +//===----------------------------------------------------------------------===// +// spirv.ShiftLeftLogical +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::ShiftLeftLogicalOp::fold( + spirv::ShiftLeftLogicalOp::FoldAdaptor adaptor) { + // x << 0 -> x + if (matchPattern(adaptor.getOperand2(), m_Zero())) { + return getOperand1(); + } + + // Unfortunately due to below undefined behaviour can't fold 0 for Base. + + // Results are computed per component, and within each component, per bit... + // + // The result is undefined if Shift is greater than or equal to the bit width + // of the components of Base. + // + // So we can use the APInt << method, but don't fold if undefined behaviour. + bool shiftToLarge = false; + auto res = constFoldBinaryOp( + adaptor.getOperands(), [&](const APInt &a, const APInt &b) { + if (shiftToLarge || b.uge(a.getBitWidth())) { + shiftToLarge = true; + return a; + } + return a << b; + }); + return shiftToLarge ? Attribute() : res; +} + +//===----------------------------------------------------------------------===// +// spirv.ShiftRightArithmetic +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::ShiftRightArithmeticOp::fold( + spirv::ShiftRightArithmeticOp::FoldAdaptor adaptor) { + // x >> 0 -> x + if (matchPattern(adaptor.getOperand2(), m_Zero())) { + return getOperand1(); + } + + // Unfortunately due to below undefined behaviour can't fold 0, -1 for Base. + + // Results are computed per component, and within each component, per bit... + // + // The result is undefined if Shift is greater than or equal to the bit width + // of the components of Base. + // + // So we can use the APInt ashr method, but don't fold if undefined behaviour. + bool shiftToLarge = false; + auto res = constFoldBinaryOp( + adaptor.getOperands(), [&](const APInt &a, const APInt &b) { + if (shiftToLarge || b.uge(a.getBitWidth())) { + shiftToLarge = true; + return a; + } + return a.ashr(b); + }); + return shiftToLarge ? Attribute() : res; +} + +//===----------------------------------------------------------------------===// +// spirv.ShiftRightLogical +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::ShiftRightLogicalOp::fold( + spirv::ShiftRightLogicalOp::FoldAdaptor adaptor) { + // x >> 0 -> x + if (matchPattern(adaptor.getOperand2(), m_Zero())) { + return getOperand1(); + } + + // Unfortunately due to below undefined behaviour can't fold 0 for Base. + + // Results are computed per component, and within each component, per bit... + // + // The result is undefined if Shift is greater than or equal to the bit width + // of the components of Base. + // + // So we can use the APInt lshr method, but don't fold if undefined behaviour. + bool shiftToLarge = false; + auto res = constFoldBinaryOp( + adaptor.getOperands(), [&](const APInt &a, const APInt &b) { + if (shiftToLarge || b.uge(a.getBitWidth())) { + shiftToLarge = true; + return a; + } + return a.lshr(b); + }); + return shiftToLarge ? Attribute() : res; +} + +//===----------------------------------------------------------------------===// +// spirv.BitwiseAndOp +//===----------------------------------------------------------------------===// + +OpFoldResult +spirv::BitwiseAndOp::fold(spirv::BitwiseAndOp::FoldAdaptor adaptor) { + // x & x -> x + if (getOperand1() == getOperand2()) { + return getOperand1(); + } + + APInt rhsMask; + if (matchPattern(adaptor.getOperand2(), m_ConstantInt(&rhsMask))) { + // x & 0 -> 0 + if (rhsMask.isZero()) + return getOperand2(); + + // x & -> x + if (rhsMask.isAllOnes()) + return getOperand1(); + + // (UConvert x : iN to iK) & -> UConvert x + if (auto zext = getOperand1().getDefiningOp()) { + int valueBits = + getElementTypeOrSelf(zext.getOperand()).getIntOrFloatBitWidth(); + if (rhsMask.zextOrTrunc(valueBits).isAllOnes()) + return getOperand1(); + } + } + + // According to the SPIR-V spec: + // + // Type is a scalar or vector of integer type. + // Results are computed per component, and within each component, per bit. + // So we can use the APInt & method. + return constFoldBinaryOp( + adaptor.getOperands(), + [](const APInt &a, const APInt &b) { return a & b; }); +} + +//===----------------------------------------------------------------------===// +// spirv.BitwiseOrOp +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::BitwiseOrOp::fold(spirv::BitwiseOrOp::FoldAdaptor adaptor) { + // x | x -> x + if (getOperand1() == getOperand2()) { + return getOperand1(); + } + + APInt rhsMask; + if (matchPattern(adaptor.getOperand2(), m_ConstantInt(&rhsMask))) { + // x | 0 -> x + if (rhsMask.isZero()) + return getOperand1(); + + // x | -> + if (rhsMask.isAllOnes()) + return getOperand2(); + } + + // According to the SPIR-V spec: + // + // Type is a scalar or vector of integer type. + // Results are computed per component, and within each component, per bit. + // So we can use the APInt | method. + return constFoldBinaryOp( + adaptor.getOperands(), + [](const APInt &a, const APInt &b) { return a | b; }); +} + +//===----------------------------------------------------------------------===// +// spirv.BitwiseXorOp +//===----------------------------------------------------------------------===// + +OpFoldResult +spirv::BitwiseXorOp::fold(spirv::BitwiseXorOp::FoldAdaptor adaptor) { + // x ^ 0 -> x + if (matchPattern(adaptor.getOperand2(), m_Zero())) { + return getOperand1(); + } + + // x ^ x -> 0 + if (getOperand1() == getOperand2()) + return Builder(getContext()).getZeroAttr(getType()); + + // According to the SPIR-V spec: + // + // Type is a scalar or vector of integer type. + // Results are computed per component, and within each component, per bit. + // So we can use the APInt ^ method. + return constFoldBinaryOp( + adaptor.getOperands(), + [](const APInt &a, const APInt &b) { return a ^ b; }); +} + //===----------------------------------------------------------------------===// // spirv.mlir.selection //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp index 3906bf74ea72235b1c29f433874bde568e2e6989..2a1d083308282a8f8a9a8be14afbe86b2c72f4d9 100644 --- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp +++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp @@ -1968,55 +1968,6 @@ LogicalResult spirv::ShiftRightLogicalOp::verify() { return verifyShiftOp(*this); } -//===----------------------------------------------------------------------===// -// spirv.BtiwiseAndOp -//===----------------------------------------------------------------------===// - -OpFoldResult -spirv::BitwiseAndOp::fold(spirv::BitwiseAndOp::FoldAdaptor adaptor) { - APInt rhsMask; - if (!matchPattern(adaptor.getOperand2(), m_ConstantInt(&rhsMask))) - return {}; - - // x & 0 -> 0 - if (rhsMask.isZero()) - return getOperand2(); - - // x & -> x - if (rhsMask.isAllOnes()) - return getOperand1(); - - // (UConvert x : iN to iK) & -> UConvert x - if (auto zext = getOperand1().getDefiningOp()) { - int valueBits = - getElementTypeOrSelf(zext.getOperand()).getIntOrFloatBitWidth(); - if (rhsMask.zextOrTrunc(valueBits).isAllOnes()) - return getOperand1(); - } - - return {}; -} - -//===----------------------------------------------------------------------===// -// spirv.BtiwiseOrOp -//===----------------------------------------------------------------------===// - -OpFoldResult spirv::BitwiseOrOp::fold(spirv::BitwiseOrOp::FoldAdaptor adaptor) { - APInt rhsMask; - if (!matchPattern(adaptor.getOperand2(), m_ConstantInt(&rhsMask))) - return {}; - - // x | 0 -> x - if (rhsMask.isZero()) - return getOperand1(); - - // x | -> - if (rhsMask.isAllOnes()) - return getOperand2(); - - return {}; -} - //===----------------------------------------------------------------------===// // spirv.ImageQuerySize //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp index 577dfe5ab2f3020ae23112232ccacb89dc2e428f..686180c09da7248f63967b1cd08423681fe8741f 100644 --- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp +++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp @@ -858,22 +858,26 @@ bool mlir::sparse_tensor::isBlockSparsity(AffineMap dimToLvl) { std::map coeffientMap; for (auto result : dimToLvl.getResults()) { if (auto binOp = dyn_cast(result)) { - auto pos = dyn_cast(binOp.getLHS()).getPosition(); - if (result.getKind() == AffineExprKind::FloorDiv) { + // Check for "dim op const". + auto dimOp = dyn_cast(binOp.getLHS()); + auto conOp = dyn_cast(binOp.getRHS()); + if (!dimOp || !conOp) + return false; + // Inspect "dim / const" or "dim % const". + auto pos = dimOp.getPosition(); + if (binOp.getKind() == AffineExprKind::FloorDiv) { // Expect only one floordiv for each dimension. if (coeffientMap.find(pos) != coeffientMap.end()) return false; - coeffientMap[pos] = - dyn_cast(binOp.getRHS()).getValue(); - } else if (result.getKind() == AffineExprKind::Mod) { + // Record coefficient of the floordiv. + coeffientMap[pos] = conOp.getValue(); + } else if (binOp.getKind() == AffineExprKind::Mod) { // Expect floordiv before mod. if (coeffientMap.find(pos) == coeffientMap.end()) return false; // Expect mod to have the same coefficient as floordiv. - if (dyn_cast(binOp.getRHS()).getValue() != - coeffientMap[pos]) { + if (conOp.getValue() != coeffientMap[pos]) return false; - } } else { return false; } diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp index 6a2d7c33356f9cb37ceec75fd83840065ecb5568..08d37b6a9656fe07352be40822f46b81b59966eb 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp @@ -147,48 +147,30 @@ static Value genSparseReducedAffineCond(OpBuilder &builder, Location loc, // Helper functions that load/store into the position buffer for slice-driven // loops. -// The sliced pointer buffer is orgnized as: -// [size, curPtr] (two metadata) + [[pLo0, pLo1, pLo2, ...], -// [pHi0, pHi1, pHi2, ...], -// [pNx0, pNx1, pNx2, ...]] +// The sliced pointer buffer is organized as: +// [[pLo0, pLo1, pLo2, ...], +// [pHi0, pHi1, pHi2, ...], +// [pNx0, pNx1, pNx2, ...]] static Value allocSlicePosBuf(OpBuilder &builder, Location loc, Value tupleCnt) { Value bufSz = MULI(tupleCnt, C_IDX(kSliceIterWidth)); // Additional two metadata {memSize, idx} at head. - bufSz = ADDI(bufSz, C_IDX(2)); return genAlloca(builder, loc, bufSz, builder.getIndexType()); } -// TODO: We should use SSA value for it. -// Gets and sets metadata. -static Value loadSlicePosPtr(OpBuilder &builder, Location loc, Value sPosBuf) { - return genIndexLoad(builder, loc, sPosBuf, C_IDX(1)); -} -static void updateSlicePosPtr(OpBuilder &builder, Location loc, Value sPosBuf, - Value pPtr) { - builder.create(loc, pPtr, sPosBuf, C_IDX(1)); -} -static Value loadSlicePosTupleNum(OpBuilder &builder, Location loc, - Value sPosBuf) { - return genIndexLoad(builder, loc, sPosBuf, C_IDX(0)); -} -static void updateSlicePosTupleNum(OpBuilder &builder, Location loc, Value num, - Value sPosBuf) { - builder.create(loc, num, sPosBuf, C_IDX(0)); -} // Gets and sets position values for slice-driven loops. enum class SlicePosKind { kLo, kHi, kNext }; static Value getSlicePosIdx(OpBuilder &builder, Location loc, Value posBuf, Value tupleIdx, SlicePosKind posKind) { Value dim = builder.create(loc, posBuf, C_IDX(0)); - Value tupleCnt = DIVUI(SUBI(dim, C_IDX(2)), C_IDX(kSliceIterWidth)); + Value tupleCnt = DIVUI(dim, C_IDX(kSliceIterWidth)); switch (posKind) { case SlicePosKind::kLo: - return ADDI(tupleIdx, C_IDX(2)); + return tupleIdx; case SlicePosKind::kHi: - return ADDI(tupleIdx, ADDI(tupleCnt, C_IDX(2))); + return ADDI(tupleIdx, tupleCnt); case SlicePosKind::kNext: - return ADDI(tupleIdx, ADDI(tupleCnt, ADDI(tupleCnt, C_IDX(2)))); + return ADDI(tupleIdx, MULI(tupleCnt, C_IDX(2))); } llvm_unreachable("unexpected kind"); } @@ -352,6 +334,9 @@ void LoopEmitter::initialize(ValueRange ts, StringAttr loopTag, bool hasOutput, this->dependentLvlMap.assign( numTensors, std::vector>>()); this->slicePosBuffer.assign(numTensors, std::vector>()); + this->sliceTupleNxStartIdx.assign(numTensors, std::vector()); + this->sliceTupleFwdCnt.assign(numTensors, std::vector()); + this->trivialSlice.assign(numTensors, std::vector()); this->sliceMeta.assign( numTensors, std::vector>>()); this->sliceStack.assign(numTensors, std::vector()); @@ -402,10 +387,13 @@ void LoopEmitter::initialize(ValueRange ts, StringAttr loopTag, bool hasOutput, dependentLvlMap[tid].assign( lvlRank, std::vector>()); slicePosBuffer[tid].assign(lvlRank, std::vector()); + sliceTupleNxStartIdx[tid].assign(lvlRank, Value()); + sliceTupleFwdCnt[tid].assign(lvlRank, Value()); + trivialSlice[tid].assign(lvlRank, false); sliceMeta[tid].assign(lvlRank, std::vector>()); sliceStack[tid].emplace_back(/*minCrd=*/Value(), /*offset=*/Value(), /*isNonEmpty*/ Value(), - std::nullopt, 0); + /*posTupleNum=*/Value(), std::nullopt, 0); if (dimGetter && !isSynTensor(tid)) { for (Level l = 0; l < lvlRank; l++) { dependentLvlMap[tid][l] = dimGetter(tid, l); @@ -814,6 +802,7 @@ std::optional LoopEmitter::genWhileLoopBody(OpBuilder &builder, assert(ivs.size() == 1); // Coord is the relative offset related to its parents. assert(sliceStack[tid].back().depth == 1 && "TODO: not yet implement"); + sliceTupleFwdCnt[tid][lvl] = SUBI(ivs[0], posits[tid][lvl]); // Update c = absOffset[lvl][depth] - absOffset[lvl][depth - 1] Value posit = ivs[0]; Value crdBuf = coordinatesBuffers[tid][lvl]; @@ -1332,6 +1321,12 @@ void LoopEmitter::enterTensorsAtDenseLvls( } else { posits[tid][lvl] = genAddress(builder, loc, tid, lvl, ADDI(info.offset, iv)); + Value fwdCnt = lvl == 0 || trivialSlice[tid][lvl] + ? C_IDX(0) + : sliceTupleFwdCnt[tid][lvl - 1]; + Value sz = sliceMeta[tid][lvl].back().first; + Value mul = MULI(fwdCnt, sz); + sliceTupleFwdCnt[tid][lvl] = ADDI(mul, iv); } levelReducedDep[tid][lvl]++; } else { @@ -1365,13 +1360,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc, assert(isDenseLT(lvlTypes[tid][lvl])); assert(*info.slicedOnLvl == lvl); (void)reduced; - // Resets slices pointers as the resolved slices are invalidated after we - // moves forward to the next slice. - invalidateSliceIterIdx(rewriter, loc, tid, lvl); info.minCrd = info.offset = info.isNonEmpty = Value(); - } else { - forwardsReducedSliceLevelTreeIt(rewriter, loc, tid, lvl, - constantIndex(rewriter, loc, 1)); } levelReducedDep[tid][lvl]--; } @@ -1451,63 +1440,6 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc, } } -void LoopEmitter::forwardsReducedSliceLevelTreeIt(OpBuilder &builder, - Location loc, TensorId tid, - Level rootLvl, Value fcnt) { - auto stt = getSparseTensorType(tensors[tid]); - - // Finds a [Lvl, leafLvl) range, and all level in between are fully reduced - // level (but not resolved). Since we forward an iterator at higher level of - // the tree, the subtree need to be pruned. - Level leafLvl = rootLvl + 1; - while (leafLvl < stt.getLvlRank() && !dependentLvlMap[tid][leafLvl].empty() && - depFullyReduced(tid, leafLvl)) { - leafLvl++; - } - - Level curLvl = rootLvl + 1; - // Prunes all denses subtree. - while (curLvl < leafLvl && isDenseLT(lvlTypes[tid][curLvl])) { - // One step forward in parent level results in forwarding `slice.size` step - // in child dense level. - auto [size, stride] = sliceMeta[tid][curLvl].back(); - assert(stride == 1 && "Not yet implemented"); - fcnt = MULI(size, fcnt); - curLvl++; - } - - Value nxPosPtr = nullptr; - if (curLvl < leafLvl) { - assert(!isDenseLT(lvlTypes[tid][curLvl])); - // The first compressed level, setting up the position pointer for it. - Value sPosBuf = slicePosBuffer[tid][curLvl].back(); - // One step forwards in the parent level result in forwarding one `segment` - // in the child sparse level. - Value pPosPtr = loadSlicePosPtr(builder, loc, sPosBuf); // previous ptr - Value cPosPtr = ADDI(fcnt, pPosPtr); // current ptr - updateSlicePosPtr(builder, loc, sPosBuf, cPosPtr); - // Loads the position pointer start for next level. - nxPosPtr = - loadSlicePos(builder, loc, sPosBuf, cPosPtr, SlicePosKind::kNext); - curLvl++; - } - - // TODO: This is not always needed, but we did it unconditionally for now for - // simplicity. - // It is only needed when `curLvl` is forwarded without traversing its child - // level (e.g., the level is in a conjunctive lattices and got pruned), such - // that the position pointer is not forwarded inside the loop. - for (; curLvl < leafLvl; curLvl++) { - assert(nxPosPtr); - if (!isDenseLT(lvlTypes[tid][curLvl])) { - Value sPosBuf = slicePosBuffer[tid][curLvl].back(); - updateSlicePosPtr(builder, loc, sPosBuf, nxPosPtr); - nxPosPtr = - loadSlicePos(builder, loc, sPosBuf, nxPosPtr, SlicePosKind::kNext); - } - } -} - void LoopEmitter::exitWhileLoop(OpBuilder &builder, Location loc, MutableArrayRef reduc) { const LoopInfo &loopInfo = loopStack.back(); @@ -1557,13 +1489,6 @@ void LoopEmitter::exitWhileLoop(OpBuilder &builder, Location loc, forwarded = CMPI(eq, coords[tid][lvl], iv); operands.push_back(SELECT(forwarded, nxPos, pos)); } - { - OpBuilder::InsertionGuard guard(builder); - auto ifOp = builder.create(loc, TypeRange{}, forwarded, - /*else=*/false); - builder.setInsertionPointToStart(&ifOp.getThenRegion().front()); - forwardsReducedSliceLevelTreeIt(builder, loc, tid, lvl, one); - } // The coordinate is invalid now. coords[tid][lvl] = nullptr; @@ -1806,7 +1731,7 @@ ValueRange LoopEmitter::genUnResolvedSliceTreeTraverse( unsigned depth = frontSlice.depth - 1; Value offset = frontSlice.offset; Value sPtrBuf = slicePosBuffer[tid][firstLvl][depth]; - Value mSz = loadSlicePosTupleNum(builder, loc, sPtrBuf); + Value mSz = frontSlice.posTupleNum; outerMost = builder.create( loc, c0, mSz, c1, innerArgs, [this, tid, firstLvl, offset, sPtrBuf, &ip, &pos, @@ -1917,7 +1842,7 @@ void LoopEmitter::genResolvedSliceBegin(OpBuilder &builder, Location loc, // Dense slice begin is trivial. sliceStack[tid].emplace_back(/*minCoord=*/c0, /*offset=*/c0, /*nonEmpty=*/constantI1(builder, loc, true), - lvl, /*depth=*/1); + c0, lvl, /*depth=*/1); return; } auto [nxSz, stride] = sliceMeta[tid][lvl][1]; @@ -1933,12 +1858,12 @@ void LoopEmitter::genResolvedSliceBegin(OpBuilder &builder, Location loc, pHi = genIndexLoad(builder, loc, positionsBuffers[tid][lvl], ADDI(posits[tid][lvl - 1], c1)); } - // Fills out pIdxBuffer[tid][lvl][0] with [/*memSize =*/4, 0, pLo, pHi] - updateSlicePosTupleNum(builder, loc, c1, sPtrBuf); - updateSlicePosPtr(builder, loc, sPtrBuf, c0); + // Fills out pIdxBuffer[tid][lvl][0] with [pLo, pHi] updateSlicePos(builder, loc, sPtrBuf, pLo, c0, SlicePosKind::kLo); updateSlicePos(builder, loc, sPtrBuf, pHi, c0, SlicePosKind::kHi); - + // Slice over a resolved parent, we only need one pair of pos hi and lo to + // specify the current slice. + Value tupleNum = c1; // This is an non empty tensor if pLo < pHi. Value isNonEmpty = CMPI(ult, pLo, pHi); // The minimal coord must be at the first on ordered level. @@ -1950,7 +1875,7 @@ void LoopEmitter::genResolvedSliceBegin(OpBuilder &builder, Location loc, // FIXME: We need the relative offset related to the base slice. Value absOffset = offsetFromMinCoord(builder, loc, minCrd, nxSz, isNonEmpty); - sliceStack[tid].emplace_back(minCrd, absOffset, isNonEmpty, lvl, + sliceStack[tid].emplace_back(minCrd, absOffset, isNonEmpty, tupleNum, lvl, /*depth=*/1); } @@ -1982,8 +1907,8 @@ void LoopEmitter::genUnResolvedSliceBegin(OpBuilder &builder, Location loc, Value remSz = sliceMeta[tid][lvl][depth + 1].first; // Dense slice begin is trivial if (isDenseLT(lvlTypes[tid][lvl])) { - sliceStack[tid].emplace_back(c0, c0, constantI1(builder, loc, false), lvl, - depth + 1); + sliceStack[tid].emplace_back(c0, c0, constantI1(builder, loc, false), c0, + lvl, depth + 1); return; } @@ -2072,26 +1997,38 @@ void LoopEmitter::genUnResolvedSliceBegin(OpBuilder &builder, Location loc, Value isNonEmpty = result[0]; Value minCrd = result[1]; // Two metadata [memSize, idx]. - // TODO: Can use an SSA value for these two metadata - updateSlicePosTupleNum(builder, loc, result[2], sPtrBuf); - updateSlicePosPtr(builder, loc, sPtrBuf, c0); // FIXME: we need the relative offset related to the base slice. Value absOffset = offsetFromMinCoord(builder, loc, minCrd, remSz, isNonEmpty); - sliceStack[tid].emplace_back(minCrd, absOffset, isNonEmpty, lvl, depth + 1); + sliceStack[tid].emplace_back(minCrd, absOffset, isNonEmpty, result[2], lvl, + depth + 1); } bool LoopEmitter::genSliceBegin(OpBuilder &builder, Location loc, TensorId tid, Level lvl) { + Value curLvlIdx = C_IDX(0); if (depFullyReduced(tid, lvl)) { - // Do not need to prepare for slice driven loop on dense level after it is - // fully reduced. + if (lvl == 0 || trivialSlice[tid][lvl]) { + sliceTupleNxStartIdx[tid][lvl] = C_IDX(0); + } else { + if (isDenseLT(lvlTypes[tid][lvl])) { + sliceTupleNxStartIdx[tid][lvl] = sliceTupleNxStartIdx[tid][lvl - 1]; + } else { + assert(isCompressedLT(lvlTypes[tid][lvl])); + curLvlIdx = ADDI(sliceTupleNxStartIdx[tid][lvl - 1], + sliceTupleFwdCnt[0][lvl - 1]); + sliceTupleNxStartIdx[tid][lvl] = + loadSlicePos(builder, loc, slicePosBuffer[tid][lvl].back(), + curLvlIdx, SlicePosKind::kNext); + } + } if (isDenseLT(lvlTypes[tid][lvl])) return true; + + Value sPosBuf = slicePosBuffer[tid][lvl].back(); // If constraints on the tensor is fully resolved. We do not need to // generates slice begin any more, instead we fall back to TACO-based // algorithm to (co)iterates over the slice. - Value sPosBuf = slicePosBuffer[tid][lvl].back(); - Value tupleIdx = loadSlicePosPtr(builder, loc, sPosBuf); + Value tupleIdx = curLvlIdx; posits[tid][lvl] = loadSlicePos(builder, loc, sPosBuf, tupleIdx, SlicePosKind::kLo); highs[tid][lvl] = @@ -2150,23 +2087,16 @@ bool LoopEmitter::genSliceBegin(OpBuilder &builder, Location loc, TensorId tid, if (sliceInfo.isInitialTensor() || (lvl >= 1 && lvlFullyResolved(tid, lvl - 1))) { // First level or previous level has been full resolved. + trivialSlice[tid][lvl] = true; genResolvedSliceBegin(builder, loc, tid, lvl); } else { // The previous level has not been full resolved. + trivialSlice[tid][lvl] = false; genUnResolvedSliceBegin(builder, loc, tid, lvl); } return false; } -void LoopEmitter::invalidateSliceIterIdx(OpBuilder &builder, Location loc, - TensorId tid, Level lvl) { - for (unsigned i = 0; i <= lvl; i++) { - if (!isDenseLT(lvlTypes[tid][i]) && !dependentLvlMap[tid][i].empty()) { - updateSlicePosPtr(builder, loc, slicePosBuffer[tid][i].back(), C_IDX(0)); - } - } -} - std::tuple LoopEmitter::genSliceNextInduction(OpBuilder &builder, Location loc, TensorId tid, Level lvl) { @@ -2191,10 +2121,6 @@ LoopEmitter::genSliceNextInduction(OpBuilder &builder, Location loc, // isNonEmpty = false; // Value absOffset = info.offset; - // Resets slices pointers as the resolved slices are invalidated after we - // moves forward to the next slice. - invalidateSliceIterIdx(builder, loc, tid, lvl); - SmallVector reduc = {info.minCrd, info.isNonEmpty, absOffset}; Value sPtrBuf = slicePosBuffer[tid][lvl][info.depth - 1]; Value fastPathP = CMPI(ugt, info.minCrd, absOffset); @@ -2221,10 +2147,10 @@ LoopEmitter::genSliceNextInduction(OpBuilder &builder, Location loc, // offset = minCrd - size + 1; // } builder.setInsertionPointToStart(&ifOp.getElseRegion().front()); - reduc[2] = absOffset; // restore value. - Value mSz = loadSlicePosTupleNum(builder, loc, sPtrBuf); // memSize - reduc[0] = lvlSizes[tid][lvl]; // next min coord - reduc[1] = constantI1(builder, loc, false); // isNonEmpty + reduc[2] = absOffset; // restore value. + Value mSz = info.posTupleNum; // tuple number. + reduc[0] = lvlSizes[tid][lvl]; // next min coord + reduc[1] = constantI1(builder, loc, false); // isNonEmpty auto loopArgs = static_cast(reduc).drop_back(); auto forOp = scf::buildLoopNest( builder, loc, c0, mSz, c1, loopArgs, diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h b/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h index 0bdd9d45e6ae6e658d7a380d7c01d085f7b959ad..fa8b0076f733b48b532f88b4d12a98fbefe6303e 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h +++ b/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h @@ -298,10 +298,10 @@ private: struct SliceInfo final { // Note that we do not need to create a actual sparse tensor slice but // instead only need to maintain the metadata of the slice. - SliceInfo(Value minCrd, Value offset, Value isNonEmpty, + SliceInfo(Value minCrd, Value offset, Value isNonEmpty, Value posTupleNum, std::optional slicedOnLvl, unsigned depth) : minCrd(minCrd), offset(offset), isNonEmpty(isNonEmpty), - slicedOnLvl(slicedOnLvl), depth(depth) { + posTupleNum(posTupleNum), slicedOnLvl(slicedOnLvl), depth(depth) { // TODO: use std::optional> assert(!slicedOnLvl || minCrd); } @@ -309,9 +309,10 @@ private: // Whether this is the tensor that has not yet been sliced. bool isInitialTensor() const { return !slicedOnLvl.has_value(); } - Value minCrd; // the minimum coordinate of the slice. - Value offset; // the *absolute* offset of the current slice. - Value isNonEmpty; // whether the slice is empty. + Value minCrd; // the minimum coordinate of the slice. + Value offset; // the *absolute* offset of the current slice. + Value isNonEmpty; // whether the slice is empty. + Value posTupleNum; // The number of position tuples used in the slice. std::optional slicedOnLvl; // the level on which the slice is done unsigned depth; // the depth (relative to dependentDimMap[tid][lvl]). }; @@ -452,11 +453,6 @@ private: return tid < lvlTypes.size() && lvl < lvlTypes[tid].size(); } - /// Forwards the (conceptual) "tree iterator" when iterating over a fully - /// reduced slice created by index-reduction. - void forwardsReducedSliceLevelTreeIt(OpBuilder &builder, Location loc, - TensorId tid, Level lvl, Value fcnt); - /// Prepares loop for iterating over `tensor[lvl]`, under the assumption /// that `tensor[0...lvl-1]` loops have already been set up. void prepareLoopOverTensorAtLvl(OpBuilder &builder, Location loc, @@ -609,11 +605,6 @@ private: void genUnResolvedSliceBegin(OpBuilder &builder, Location loc, TensorId tid, Level lvl); - /// Invalidates the index kept in slice postion buffers (by setting it to - /// zero). - /// TODO: We should instead use an SSA value for the index. - void invalidateSliceIterIdx(OpBuilder &builder, Location loc, TensorId tid, - Level lvl); /// Generates code to get the first non-empty slice of tid on lvl. /// return true if has already been resolved. bool genSliceBegin(OpBuilder &builder, Location loc, TensorId tid, Level lvl); @@ -650,17 +641,6 @@ private: std::vector> lvlTypes; // Sparse iteration information for each `(TensorId, Level)` pair. // These arrays are updated to remain current within the current loop. - // TODO: Clarify which of these are indexed by dstLvl vs srcLvl. - // - /// The collection of positions for a given element (one such collection - /// for each tensor). This is the position analogue of the "coords" - /// naming convention. - /// - /// FIXME: [CLARIFY_POSITS_LVL] It's unclear which levels are used - /// to index the `posits` array. On the one hand `genSparseCrd` - /// uses dstLvl; on the other hand `enterLoopOverTensorAtLvl`, - /// `prepareLoopOverTensorAtLvl`, and `enterCoIterationOverTensorsAtLvls` - /// uses srcLvl. So which is it? std::vector> posits; /// The collection of coordinates for a given element (one such /// collection for each tensor). @@ -693,6 +673,9 @@ private: // But they always starts with the first pidx pointing to coord > slice.offset // to avoid iteration from the beginning. std::vector>> slicePosBuffer; + std::vector> sliceTupleNxStartIdx; + std::vector> sliceTupleFwdCnt; + std::vector> trivialSlice; // The (size, stride) for each conceptual slice used for index reduction // loops. @@ -704,10 +687,6 @@ private: // sliceStack[tid] holds the generated slice stack on tid. std::vector> sliceStack; - /// TODO: not yet used, it should track the current level for each tensor - /// to help eliminate `lvls` paramters from above APIs. - /// std::vector curLvl; - // // Fields which have at most `numLoops` many entries. // diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp index 463a49f52283a735507828e44ea562a00fbf4945..cdbf4f048a00f38ebcc36f9807b4e2364efd69df 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp @@ -952,9 +952,9 @@ createQuickSort(OpBuilder &builder, ModuleOp module, func::FuncOp func, Value cond = builder.create(loc, arith::CmpIPredicate::ule, lenLow, lenHigh); + Value c0 = constantIndex(builder, loc, 0); scf::IfOp ifOp = builder.create(loc, types, cond, /*else=*/true); - Value c0 = constantIndex(builder, loc, 0); auto mayRecursion = [&](Value low, Value high, Value len) { Value cond = builder.create(loc, arith::CmpIPredicate::ne, len, c0); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp index 927fc71f84a85a8b9cce974f2ef30e12782a9161..5155cab772d4403eaf96be3d892e4af9aa258440 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp @@ -309,6 +309,10 @@ static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc, // } Value upper = irMap.lookup(forallOp.getUpperBound()[0]); scf::ForOp forOp = rewriter.create(loc, row, upper, inc); + // The scf.for builder creates an empty block. scf.for does not allow multiple + // blocks in its region, so delete the block before `cloneRegionBefore` adds + // an additional block. + rewriter.eraseBlock(forOp.getBody()); rewriter.cloneRegionBefore(forallOp.getRegion(), forOp.getRegion(), forOp.getRegion().begin(), irMap); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp index c94ef8b96287766c4b4bb7b7c4dfad485df50756..488079cfe4e32ddb0a86f42ecd4a0223db43f77e 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp @@ -38,16 +38,22 @@ struct DemapInsRewriter : public OpRewritePattern { LogicalResult matchAndRewrite(SourceOp op, PatternRewriter &rewriter) const override { Location loc = op.getLoc(); + // Demaps non-trivial inputs. + bool changed = false; SmallVector deMappedIns(op->getOperands()); - for (Value &in : deMappedIns) - if (auto stt = tryGetSparseTensorType(in); stt && !stt->isIdentity()) + for (Value &in : deMappedIns) { + if (auto stt = tryGetSparseTensorType(in); stt && !stt->isIdentity()) { in = rewriter.create(loc, stt->getDemappedType(), in); + changed = true; + } + } // CRTP call. OpAdaptor adaptor(deMappedIns, op); - return static_cast(this)->rewriteOp(op, adaptor, - rewriter); + LogicalResult status = + static_cast(this)->rewriteOp(op, adaptor, rewriter); + return changed ? success() : status; } }; @@ -452,11 +458,13 @@ struct GenericOpScheduler : public OpRewritePattern { } // Marks the GenericOp to avoid recursive matching. - linalgOp->setAttr(sorted, rewriter.getBoolAttr(true)); + rewriter.updateRootInPlace(linalgOp, [&]() { + linalgOp->setAttr(sorted, rewriter.getBoolAttr(true)); + }); // Already sorted. if (order.isIdentity()) - return failure(); + return success(); assert(order.isPermutation()); // `order` is orignial loop -> sorted loop map diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp index 6c9adf9fa21a0cba67fa3b6722d68e0680529309..992be434fc62314ff9c88930032e3ce6106f2701 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp @@ -673,25 +673,35 @@ static void genInvariants(CodegenEnv &env, OpBuilder &builder, ExprId exp, // All exhausted at current level. if (!isCurrentLoop) return; + // Generate code for a scalarized reduction or invariant. Note that + // because custom reduction lhs may occur several times in the IR, + // we have a built-in safety for only initializing and wrapping-up + // the scalarized reduction once. OpOperand *lhs = op.getDpsInitOperand(0); if (lhs == &t) { // Start or end a scalarized reduction. if (isStart) { - Value load = env.isCustomReduc() ? env.getCustomRedId() - : genTensorLoad(env, builder, exp); - env.startReduc(exp, load); + if (env.isCustomReduc()) { + if (!env.isReduc()) + env.startReduc(exp, env.getCustomRedId()); + } else { + env.startReduc(exp, genTensorLoad(env, builder, exp)); + } if (env.hasSparseOutput()) env.setValidLexInsert(constantI1(builder, env.op().getLoc(), false)); } else { - genTensorStore(env, builder, exp, env.endReduc()); - env.clearValidLexInsert(); + if (!env.isCustomReduc() || env.isReduc()) + genTensorStore(env, builder, exp, env.endReduc()); + if (env.hasSparseOutput()) + env.clearValidLexInsert(); } } else { // Start or end loop invariant hoisting of a tensor load. - if (isStart) + if (isStart) { env.merger().setExprValue(exp, genTensorLoad(env, builder, exp)); - else + } else { env.merger().clearExprValue(exp); + } } } else if (env.exp(exp).kind != TensorExp::Kind::kInvariant && env.exp(exp).kind != TensorExp::Kind::kLoopVar && diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp index 55f813df78b85710d1105b2d1cea51c3dc95bea3..ec4c41c0000a9ccf728b8ec9899a38bdf523e115 100644 --- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp +++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp @@ -168,6 +168,41 @@ static llvm::SmallBitVector getDroppedDims(ArrayRef reducedShape, return droppedDims; } +/// Given a ranked tensor type and a range of values that defines its dynamic +/// dimension sizes, turn all dynamic sizes that have a constant value into +/// static dimension sizes. +static RankedTensorType +foldDynamicToStaticDimSizes(RankedTensorType type, ValueRange dynamicSizes, + SmallVector &foldedDynamicSizes) { + SmallVector staticShape(type.getShape().begin(), + type.getShape().end()); + assert(type.getNumDynamicDims() == + static_cast(dynamicSizes.size()) && + "incorrect number of dynamic sizes"); + + // Compute new static and dynamic sizes. + unsigned ctr = 0; + for (int64_t i = 0, e = type.getRank(); i < e; ++i) { + if (type.isDynamicDim(i)) { + Value dynamicSize = dynamicSizes[ctr++]; + std::optional cst = getConstantIntValue(dynamicSize); + if (cst.has_value()) { + // Dynamic size must be non-negative. + if (cst.value() < 0) { + foldedDynamicSizes.push_back(dynamicSize); + continue; + } + staticShape[i] = *cst; + } else { + foldedDynamicSizes.push_back(dynamicSize); + } + } + } + + return RankedTensorType::get(staticShape, type.getElementType(), + type.getEncoding()); +} + //===----------------------------------------------------------------------===// // BitcastOp //===----------------------------------------------------------------------===// @@ -889,37 +924,16 @@ struct ReplaceEmptyTensorStaticShapeDims : OpRewritePattern { LogicalResult matchAndRewrite(EmptyOp op, PatternRewriter &rewriter) const override { - SmallVector staticShape(op.getType().getShape().begin(), - op.getType().getShape().end()); - SmallVector dynamicSizes; - - // Compute new static and dynamic sizes. - unsigned ctr = 0; - bool changedType = false; - for (int64_t i = 0; i < op.getType().getRank(); ++i) { - if (op.getType().isDynamicDim(i)) { - Value dynamicSize = op.getDynamicSizes()[ctr++]; - std::optional cst = getConstantIntValue(dynamicSize); - if (cst.has_value()) { - // dynamic size must be non-negative. - if (cst.value() < 0) - return failure(); - staticShape[i] = *cst; - changedType = true; - } else { - dynamicSizes.push_back(dynamicSize); - } - } - } + SmallVector foldedDynamicSizes; + RankedTensorType foldedTensorType = foldDynamicToStaticDimSizes( + op.getType(), op.getDynamicSizes(), foldedDynamicSizes); // Stop here if no dynamic size was promoted to static. - if (!changedType) + if (foldedTensorType == op.getType()) return failure(); - auto tensorType = RankedTensorType::get( - staticShape, op.getType().getElementType(), op.getType().getEncoding()); - auto newOp = - rewriter.create(op.getLoc(), tensorType, dynamicSizes); + auto newOp = rewriter.create(op.getLoc(), foldedTensorType, + foldedDynamicSizes); rewriter.replaceOpWithNewOp(op, op.getType(), newOp); return success(); } @@ -1347,28 +1361,6 @@ LogicalResult GenerateOp::reifyResultShapes( return success(); } -/// Extract operands and shape from a tensor with dynamic extents. -static void operandsAndShape(TensorType resultType, - Operation::operand_range dynamicExtents, - SmallVectorImpl &newOperands, - SmallVectorImpl &newShape) { - auto operandsIt = dynamicExtents.begin(); - for (int64_t dim : resultType.getShape()) { - if (!ShapedType::isDynamic(dim)) { - newShape.push_back(dim); - continue; - } - APInt index; - if (!matchPattern(*operandsIt, m_ConstantInt(&index))) { - newShape.push_back(ShapedType::kDynamic); - newOperands.push_back(*operandsIt++); - continue; - } - newShape.push_back(index.getSExtValue()); - operandsIt++; - } -} - LogicalResult GenerateOp::verify() { // Ensure that the tensor type has as many dynamic dimensions as are // specified by the operands. @@ -1376,14 +1368,6 @@ LogicalResult GenerateOp::verify() { if (getNumOperands() != resultType.getNumDynamicDims()) return emitError("must have as many index operands as dynamic extents " "in the result type"); - // Ensure operands are non-negative. - SmallVector newOperands; - SmallVector newShape; - operandsAndShape(resultType, getDynamicExtents(), newOperands, newShape); - for (int64_t newdim : newShape) { - if (newdim < 0 && !ShapedType::isDynamic(newdim)) - return emitError("tensor dimensions must be non-negative"); - } return success(); } @@ -1433,34 +1417,24 @@ namespace { struct StaticTensorGenerate : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(GenerateOp tensorFromElements, + LogicalResult matchAndRewrite(GenerateOp generateOp, PatternRewriter &rewriter) const final { - auto resultType = - llvm::cast(tensorFromElements.getResult().getType()); + SmallVector foldedDynamicSizes; + RankedTensorType foldedTensorType = foldDynamicToStaticDimSizes( + generateOp.getType(), generateOp.getDynamicExtents(), + foldedDynamicSizes); - if (resultType.hasStaticShape()) - return failure(); - - Operation::operand_range dynamicExtents = - tensorFromElements.getDynamicExtents(); - SmallVector newOperands; - SmallVector newShape; - operandsAndShape(resultType, dynamicExtents, newOperands, newShape); - - if (!hasValidSizesOffsets(newShape)) - return failure(); - - if (newOperands.size() == tensorFromElements.getDynamicExtents().size()) + // Stop here if no dynamic size was promoted to static. + if (foldedTensorType == generateOp.getType()) return failure(); - auto loc = tensorFromElements.getLoc(); - auto newOp = rewriter.create( - loc, RankedTensorType::get(newShape, resultType.getElementType()), - newOperands); - rewriter.inlineRegionBefore(tensorFromElements.getBody(), newOp.getBody(), + auto loc = generateOp.getLoc(); + auto newOp = + rewriter.create(loc, foldedTensorType, foldedDynamicSizes); + rewriter.inlineRegionBefore(generateOp.getBody(), newOp.getBody(), newOp.getBody().begin()); - rewriter.replaceOpWithNewOp(tensorFromElements, resultType, - newOp); + rewriter.replaceOpWithNewOp(generateOp, + generateOp.getType(), newOp); return success(); } }; diff --git a/mlir/lib/Dialect/Utils/IndexingUtils.cpp b/mlir/lib/Dialect/Utils/IndexingUtils.cpp index f4e29539214b4b6f74acd59de5b8dd653f1d3f5e..bb8a0d5912d7c1508bdf32d26000ceaeac8ef2b9 100644 --- a/mlir/lib/Dialect/Utils/IndexingUtils.cpp +++ b/mlir/lib/Dialect/Utils/IndexingUtils.cpp @@ -70,7 +70,7 @@ SmallVector delinearizeImpl(ExprType linearIndex, //===----------------------------------------------------------------------===// SmallVector mlir::computeSuffixProduct(ArrayRef sizes) { - assert(llvm::all_of(sizes, [](int64_t s) { return s > 0; }) && + assert(llvm::all_of(sizes, [](int64_t s) { return s >= 0; }) && "sizes must be nonnegative"); int64_t unit = 1; return ::computeSuffixProductImpl(sizes, unit); diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp index c462b23e1133fc9d999380c7415082b515271c95..540959b486db9c5c675726613499ddf9ffbaf576 100644 --- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp +++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp @@ -3815,6 +3815,11 @@ ParseResult TransferReadOp::parse(OpAsmParser &parser, OperationState &result) { if (llvm::dyn_cast(shapedType.getElementType())) return parser.emitError( maskInfo.location, "does not support masks with vector element type"); + if (vectorType.getRank() != permMap.getNumResults()) { + return parser.emitError(typesLoc, + "expected the same rank for the vector and the " + "results of the permutation map"); + } // Instead of adding the mask type as an op type, compute it based on the // vector type and the permutation map (to keep the type signature small). auto maskType = inferTransferOpMaskType(vectorType, permMap); @@ -4181,6 +4186,11 @@ ParseResult TransferWriteOp::parse(OpAsmParser &parser, if (llvm::dyn_cast(shapedType.getElementType())) return parser.emitError( maskInfo.location, "does not support masks with vector element type"); + if (vectorType.getRank() != permMap.getNumResults()) { + return parser.emitError(typesLoc, + "expected the same rank for the vector and the " + "results of the permutation map"); + } auto maskType = inferTransferOpMaskType(vectorType, permMap); if (parser.resolveOperand(maskInfo, maskType, result.operands)) return failure(); diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp index 4d43a76c4a4efcc37279f52cc62db7542c08a72f..97f6caca1b25cccc99067294a2a25f2d68ac5ac0 100644 --- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp @@ -334,24 +334,22 @@ public: return rewriter.notifyMatchFailure( op, "Options specifies lowering to shuffle"); - if (vectorTransformOptions.useShapeCast) { - // Replace: - // vector.transpose %0, [1, 0] : vector> to - // vector<1xnxelty> - // with: - // vector.shape_cast %0 : vector> to vector<1xnxelty> - // - // Source with leading unit dim (inverse) is also replaced. Unit dim must - // be fixed. Non-unit can be scalable. - if (resType.getRank() == 2 && - ((resType.getShape().front() == 1 && - !resType.getScalableDims().front()) || - (resType.getShape().back() == 1 && - !resType.getScalableDims().back())) && - transp == ArrayRef({1, 0})) { - rewriter.replaceOpWithNewOp(op, resType, input); - return success(); - } + // Replace: + // vector.transpose %0, [1, 0] : vector> to + // vector<1xnxelty> + // with: + // vector.shape_cast %0 : vector> to vector<1xnxelty> + // + // Source with leading unit dim (inverse) is also replaced. Unit dim must + // be fixed. Non-unit can be scalable. + if (resType.getRank() == 2 && + ((resType.getShape().front() == 1 && + !resType.getScalableDims().front()) || + (resType.getShape().back() == 1 && + !resType.getScalableDims().back())) && + transp == ArrayRef({1, 0})) { + rewriter.replaceOpWithNewOp(op, resType, input); + return success(); } if (inputType.isScalable()) diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp index 4b76dcf7f8a9f7c21c5b5526dbef8b7639aa8adc..1f7cbf349255d5f40d09887e33e88aa26963318e 100644 --- a/mlir/lib/IR/AsmPrinter.cpp +++ b/mlir/lib/IR/AsmPrinter.cpp @@ -16,7 +16,9 @@ #include "mlir/IR/AsmState.h" #include "mlir/IR/Attributes.h" #include "mlir/IR/Builders.h" +#include "mlir/IR/BuiltinAttributes.h" #include "mlir/IR/BuiltinDialect.h" +#include "mlir/IR/BuiltinTypeInterfaces.h" #include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/Dialect.h" #include "mlir/IR/DialectImplementation.h" @@ -27,6 +29,7 @@ #include "mlir/IR/Operation.h" #include "mlir/IR/Verifier.h" #include "llvm/ADT/APFloat.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLExtras.h" @@ -44,6 +47,7 @@ #include "llvm/Support/SaveAndRestore.h" #include "llvm/Support/Threading.h" #include "llvm/Support/raw_ostream.h" +#include #include #include @@ -425,6 +429,8 @@ public: void popCyclicPrinting(); + void printDimensionList(ArrayRef shape); + protected: void printOptionalAttrDict(ArrayRef attrs, ArrayRef elidedAttrs = {}, @@ -1860,6 +1866,20 @@ private: // Allow direct access to the impl fields. friend AsmState; }; + +template +void printDimensionList(raw_ostream &stream, Range &&shape) { + llvm::interleave( + shape, stream, + [&stream](const auto &dimSize) { + if (ShapedType::isDynamic(dimSize)) + stream << "?"; + else + stream << dimSize; + }, + "x"); +} + } // namespace detail } // namespace mlir @@ -2576,13 +2596,9 @@ void AsmPrinter::Impl::printTypeImpl(Type type) { }) .Case([&](RankedTensorType tensorTy) { os << "tensor<"; - for (int64_t dim : tensorTy.getShape()) { - if (ShapedType::isDynamic(dim)) - os << '?'; - else - os << dim; + printDimensionList(tensorTy.getShape()); + if (!tensorTy.getShape().empty()) os << 'x'; - } printType(tensorTy.getElementType()); // Only print the encoding attribute value if set. if (tensorTy.getEncoding()) { @@ -2598,13 +2614,9 @@ void AsmPrinter::Impl::printTypeImpl(Type type) { }) .Case([&](MemRefType memrefTy) { os << "memref<"; - for (int64_t dim : memrefTy.getShape()) { - if (ShapedType::isDynamic(dim)) - os << '?'; - else - os << dim; + printDimensionList(memrefTy.getShape()); + if (!memrefTy.getShape().empty()) os << 'x'; - } printType(memrefTy.getElementType()); MemRefLayoutAttrInterface layout = memrefTy.getLayout(); if (!llvm::isa(layout) || !layout.isIdentity()) { @@ -2735,6 +2747,10 @@ LogicalResult AsmPrinter::Impl::pushCyclicPrinting(const void *opaquePointer) { void AsmPrinter::Impl::popCyclicPrinting() { state.popCyclicPrinting(); } +void AsmPrinter::Impl::printDimensionList(ArrayRef shape) { + detail::printDimensionList(os, shape); +} + //===--------------------------------------------------------------------===// // AsmPrinter //===--------------------------------------------------------------------===// @@ -2800,6 +2816,10 @@ void AsmPrinter::printResourceHandle(const AsmDialectResourceHandle &resource) { impl->printResourceHandle(resource); } +void AsmPrinter::printDimensionList(ArrayRef shape) { + detail::printDimensionList(getStream(), shape); +} + LogicalResult AsmPrinter::pushCyclicPrinting(const void *opaquePointer) { return impl->pushCyclicPrinting(opaquePointer); } @@ -3911,3 +3931,47 @@ void Block::printAsOperand(raw_ostream &os, AsmState &state) { OperationPrinter printer(os, state.getImpl()); printer.printBlockName(this); } + +//===--------------------------------------------------------------------===// +// Custom printers +//===--------------------------------------------------------------------===// +namespace mlir { + +void printDimensionList(OpAsmPrinter &printer, Operation *op, + ArrayRef dimensions) { + if (dimensions.empty()) + printer << "["; + printer.printDimensionList(dimensions); + if (dimensions.empty()) + printer << "]"; +} + +ParseResult parseDimensionList(OpAsmParser &parser, + DenseI64ArrayAttr &dimensions) { + // Empty list case denoted by "[]". + if (succeeded(parser.parseOptionalLSquare())) { + if (failed(parser.parseRSquare())) { + return parser.emitError(parser.getCurrentLocation()) + << "Failed parsing dimension list."; + } + dimensions = + DenseI64ArrayAttr::get(parser.getContext(), ArrayRef()); + return success(); + } + + // Non-empty list case. + SmallVector shapeArr; + if (failed(parser.parseDimensionList(shapeArr, true, false))) { + return parser.emitError(parser.getCurrentLocation()) + << "Failed parsing dimension list."; + } + if (shapeArr.empty()) { + return parser.emitError(parser.getCurrentLocation()) + << "Failed parsing dimension list. Did you mean an empty list? It " + "must be denoted by \"[]\"."; + } + dimensions = DenseI64ArrayAttr::get(parser.getContext(), shapeArr); + return success(); +} + +} // namespace mlir diff --git a/mlir/lib/Pass/Pass.cpp b/mlir/lib/Pass/Pass.cpp index 658f8844b428df1d76bb299a4b4503e583f2035d..810d6a357d52c047f7a840d873b9369d88e748eb 100644 --- a/mlir/lib/Pass/Pass.cpp +++ b/mlir/lib/Pass/Pass.cpp @@ -36,11 +36,21 @@ using namespace mlir::detail; // PassExecutionAction //===----------------------------------------------------------------------===// +PassExecutionAction::PassExecutionAction(ArrayRef irUnits, + const Pass &pass) + : Base(irUnits), pass(pass) {} + void PassExecutionAction::print(raw_ostream &os) const { os << llvm::formatv("`{0}` running `{1}` on Operation `{2}`", tag, pass.getName(), getOp()->getName()); } +Operation *PassExecutionAction::getOp() const { + ArrayRef irUnits = getContextIRUnits(); + return irUnits.empty() ? nullptr + : llvm::dyn_cast_if_present(irUnits[0]); +} + //===----------------------------------------------------------------------===// // Pass //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Pass/PassDetail.h b/mlir/lib/Pass/PassDetail.h index 727607146a68c6b6cc6b478551b955127d993c2c..0e964b6d6d36bc7589dc0950cf2a991b1de4637f 100644 --- a/mlir/lib/Pass/PassDetail.h +++ b/mlir/lib/Pass/PassDetail.h @@ -15,26 +15,6 @@ #include "llvm/Support/FormatVariadic.h" namespace mlir { -/// Encapsulate the "action" of executing a single pass, used for the MLIR -/// tracing infrastructure. -struct PassExecutionAction : public tracing::ActionImpl { - using Base = tracing::ActionImpl; - PassExecutionAction(ArrayRef irUnits, const Pass &pass) - : Base(irUnits), pass(pass) {} - static constexpr StringLiteral tag = "pass-execution"; - void print(raw_ostream &os) const override; - const Pass &getPass() const { return pass; } - Operation *getOp() const { - ArrayRef irUnits = getContextIRUnits(); - return irUnits.empty() ? nullptr - : llvm::dyn_cast_if_present(irUnits[0]); - } - -public: - const Pass &pass; - Operation *op; -}; - namespace detail { //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp index 4bdffa572e31a98ad70a2f9a63c715d3138030d2..7c51ee7420f9b3a1599254afbe0bf6f3c535b090 100644 --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -619,6 +619,19 @@ void ModuleImport::setNonDebugMetadataAttrs(llvm::Instruction *inst, } } +void ModuleImport::setIntegerOverflowFlagsAttr(llvm::Instruction *inst, + Operation *op) const { + auto iface = cast(op); + + IntegerOverflowFlags value = {}; + value = bitEnumSet(value, IntegerOverflowFlags::nsw, inst->hasNoSignedWrap()); + value = + bitEnumSet(value, IntegerOverflowFlags::nuw, inst->hasNoUnsignedWrap()); + + auto attr = IntegerOverflowFlagsAttr::get(op->getContext(), value); + iface->setAttr(iface.getIntegerOverflowAttrName(), attr); +} + void ModuleImport::setFastmathFlagsAttr(llvm::Instruction *inst, Operation *op) const { auto iface = cast(op); diff --git a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp index 8e2bfe557c555f38aa99c8fb9fa0d075d44b98ac..7decbce018a878dac38d30ec4938f1e103d6547e 100644 --- a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp +++ b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp @@ -581,9 +581,6 @@ void GreedyPatternRewriteDriver::notifyOperationReplaced( }); if (config.listener) config.listener->notifyOperationReplaced(op, replacement); - for (auto result : op->getResults()) - for (auto *user : result.getUsers()) - addToWorklist(user); } LogicalResult GreedyPatternRewriteDriver::notifyMatchFailure( diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt index 55731943fb78de476361c85f1d4060e5655062a0..585918afc26335905d23ccf7b8abb3b7e477801e 100644 --- a/mlir/python/CMakeLists.txt +++ b/mlir/python/CMakeLists.txt @@ -335,7 +335,8 @@ declare_mlir_dialect_python_bindings( TD_FILE dialects/OpenMPOps.td SOURCES dialects/openmp.py - DIALECT_NAME omp) + DIALECT_NAME omp + DEPENDS omp_common_td) declare_mlir_dialect_python_bindings( ADD_TO_PARENT MLIRPythonSources.Dialects diff --git a/mlir/python/mlir/dialects/_ods_common.py b/mlir/python/mlir/dialects/_ods_common.py index 60ce83c09f1717ece0053f356444398302a37c87..1685124fbccdc9ff4eb6fdb4d54b2590296ea540 100644 --- a/mlir/python/mlir/dialects/_ods_common.py +++ b/mlir/python/mlir/dialects/_ods_common.py @@ -134,3 +134,7 @@ def get_op_result_or_op_results( # see the typing.Type doc string. _U = _TypeVar("_U", bound=_cext.ir.Value) SubClassValueT = _Type[_U] + +ResultValueTypeTuple = _cext.ir.Operation, _cext.ir.OpView, _cext.ir.Value +ResultValueT = _Union[ResultValueTypeTuple] +VariadicResultValueT = _Union[ResultValueT, _Sequence[ResultValueT]] diff --git a/mlir/python/mlir/dialects/affine.py b/mlir/python/mlir/dialects/affine.py index 26e827009bc04257399999f4aff532bfbf5cbe5a..913cea61105cee87ad8fcef3d3bb1f9adc9d4957 100644 --- a/mlir/python/mlir/dialects/affine.py +++ b/mlir/python/mlir/dialects/affine.py @@ -3,8 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception from ._affine_ops_gen import * -from ._affine_ops_gen import _Dialect, AffineForOp -from .arith import constant +from ._affine_ops_gen import _Dialect try: from ..ir import * @@ -12,6 +11,9 @@ try: get_op_result_or_value as _get_op_result_or_value, get_op_results_or_values as _get_op_results_or_values, _cext as _ods_cext, + ResultValueTypeTuple as _ResultValueTypeTuple, + ResultValueT as _ResultValueT, + VariadicResultValueT as _VariadicResultValueT, ) except ImportError as e: raise RuntimeError("Error loading imports from extension module") from e @@ -21,17 +23,17 @@ from typing import Optional, Sequence, Union @_ods_cext.register_operation(_Dialect, replace=True) class AffineForOp(AffineForOp): - """Specialization for the Affine for op class""" + """Specialization for the Affine for op class.""" def __init__( self, - lower_bound, - upper_bound, - step, - iter_args: Optional[Union[Operation, OpView, Sequence[Value]]] = None, + lower_bound: Union[int, _ResultValueT, AffineMap], + upper_bound: Optional[Union[int, _ResultValueT, AffineMap]], + step: Optional[Union[int, Attribute]] = None, + iter_args: Optional[_ResultValueT] = None, *, - lower_bound_operands=[], - upper_bound_operands=[], + lower_bound_operands: Optional[_VariadicResultValueT] = None, + upper_bound_operands: Optional[_VariadicResultValueT] = None, loc=None, ip=None, ): @@ -43,25 +45,57 @@ class AffineForOp(AffineForOp): - `iter_args` is a list of additional loop-carried arguments or an operation producing them as results. - `lower_bound_operands` is the list of arguments to substitute the dimensions, - then symbols in the `lower_bound` affine map, in an increasing order + then symbols in the `lower_bound` affine map, in an increasing order. - `upper_bound_operands` is the list of arguments to substitute the dimensions, - then symbols in the `upper_bound` affine map, in an increasing order + then symbols in the `upper_bound` affine map, in an increasing order. """ + if lower_bound_operands is None: + lower_bound_operands = [] + if upper_bound_operands is None: + upper_bound_operands = [] + + if step is None: + step = 1 + + bounds_operands = [lower_bound_operands, upper_bound_operands] + bounds = [lower_bound, upper_bound] + bounds_names = ["lower", "upper"] + for i, name in enumerate(bounds_names): + if isinstance(bounds[i], int): + bounds[i] = AffineMap.get_constant(bounds[i]) + elif isinstance(bounds[i], _ResultValueTypeTuple): + if len(bounds_operands[i]): + raise ValueError( + f"Either a concrete {name} bound or an AffineMap in combination " + f"with {name} bound operands, but not both, is supported." + ) + if ( + isinstance(bounds[i], (OpView, Operation)) + and len(bounds[i].results) > 1 + ): + raise ValueError( + f"Only a single concrete value is supported for {name} bound." + ) + + bounds_operands[i].append(_get_op_result_or_value(bounds[i])) + bounds[i] = AffineMap.get_identity(1) + + if not isinstance(bounds[i], AffineMap): + raise ValueError( + f"{name} bound must be int | ResultValueT | AffineMap." + ) + if len(bounds_operands[i]) != bounds[i].n_inputs: + raise ValueError( + f"Wrong number of {name} bound operands passed to AffineForOp; " + + f"Expected {bounds[i].n_inputs}, got {len(bounds_operands[i])}." + ) + + lower_bound, upper_bound = bounds + if iter_args is None: iter_args = [] iter_args = _get_op_results_or_values(iter_args) - if len(lower_bound_operands) != lower_bound.n_inputs: - raise ValueError( - f"Wrong number of lower bound operands passed to AffineForOp. " - + "Expected {lower_bound.n_symbols}, got {len(lower_bound_operands)}." - ) - - if len(upper_bound_operands) != upper_bound.n_inputs: - raise ValueError( - f"Wrong number of upper bound operands passed to AffineForOp. " - + "Expected {upper_bound.n_symbols}, got {len(upper_bound_operands)}." - ) results = [arg.type for arg in iter_args] super().__init__( @@ -71,7 +105,7 @@ class AffineForOp(AffineForOp): inits=list(iter_args), lowerBoundMap=AffineMapAttr.get(lower_bound), upperBoundMap=AffineMapAttr.get(upper_bound), - step=IntegerAttr.get(IndexType.get(), step), + step=step, loc=loc, ip=ip, ) @@ -98,37 +132,18 @@ class AffineForOp(AffineForOp): def for_( start, - stop=None, + stop, step=None, iter_args: Optional[Sequence[Value]] = None, *, loc=None, ip=None, ): - if step is None: - step = 1 - if stop is None: - stop = start - start = 0 - params = [start, stop] - for i, p in enumerate(params): - if isinstance(p, int): - p = constant(IntegerAttr.get(IndexType.get(), p)) - elif isinstance(p, float): - raise ValueError(f"{p=} must be int.") - params[i] = p - - start, stop = params - s0 = AffineSymbolExpr.get(0) - lbmap = AffineMap.get(0, 1, [s0]) - ubmap = AffineMap.get(0, 1, [s0]) for_op = AffineForOp( - lbmap, - ubmap, + start, + stop, step, iter_args=iter_args, - lower_bound_operands=[start], - upper_bound_operands=[stop], loc=loc, ip=ip, ) diff --git a/mlir/test/CAPI/ir.c b/mlir/test/CAPI/ir.c index 315458a08b613e0592b3d93c92470e471e6d0953..a9850c0a132e757f278eabeb0192bb9f3bccb005 100644 --- a/mlir/test/CAPI/ir.c +++ b/mlir/test/CAPI/ir.c @@ -1970,6 +1970,15 @@ int testOperands(void) { fprintf(stderr, "\n"); // CHECK: Second replacement use owner: "dummy.op2" + MlirOpOperand use5 = mlirValueGetFirstUse(constTwoValue); + MlirOpOperand use6 = mlirOpOperandGetNextUse(use5); + if (!mlirValueEqual(mlirOpOperandGetValue(use5), + mlirOpOperandGetValue(use6))) { + fprintf(stderr, + "ERROR: First and second operand should share the same value\n"); + return 5; + } + mlirOperationDestroy(op); mlirOperationDestroy(op2); mlirOperationDestroy(constZero); diff --git a/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir b/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir index 4837a0cce634a66f45a3a1e694225c1e49d10fc4..f0c4512cbfdcc7bb6ba4c48ac21244635b3b1e1d 100644 --- a/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir +++ b/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir @@ -1,5 +1,7 @@ // RUN: mlir-opt %s -convert-math-to-libm -canonicalize | FileCheck %s +// CHECK-DAG: @acos(f64) -> f64 attributes {llvm.readnone} +// CHECK-DAG: @acosf(f32) -> f32 attributes {llvm.readnone} // CHECK-DAG: @atan(f64) -> f64 attributes {llvm.readnone} // CHECK-DAG: @atanf(f32) -> f32 attributes {llvm.readnone} // CHECK-DAG: @erf(f64) -> f64 attributes {llvm.readnone} @@ -29,6 +31,43 @@ // CHECK-DAG: @ceil(f64) -> f64 attributes {llvm.readnone} // CHECK-DAG: @ceilf(f32) -> f32 attributes {llvm.readnone} +// CHECK-LABEL: func @acos_caller +// CHECK-SAME: %[[FLOAT:.*]]: f32 +// CHECK-SAME: %[[DOUBLE:.*]]: f64 +func.func @acos_caller(%float: f32, %double: f64) -> (f32, f64) { + // CHECK-DAG: %[[FLOAT_RESULT:.*]] = call @acosf(%[[FLOAT]]) : (f32) -> f32 + %float_result = math.acos %float : f32 + // CHECK-DAG: %[[DOUBLE_RESULT:.*]] = call @acos(%[[DOUBLE]]) : (f64) -> f64 + %double_result = math.acos %double : f64 + // CHECK: return %[[FLOAT_RESULT]], %[[DOUBLE_RESULT]] + return %float_result, %double_result : f32, f64 +} + +// CHECK-LABEL: func @acos_vec_caller( +// CHECK-SAME: %[[VAL_0:.*]]: vector<2xf32>, +// CHECK-SAME: %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) { +// CHECK-DAG: %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32> +// CHECK-DAG: %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64> +// CHECK: %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32> +// CHECK: %[[OUT0_F32:.*]] = call @acosf(%[[IN0_F32]]) : (f32) -> f32 +// CHECK: %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32> +// CHECK: %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32> +// CHECK: %[[OUT1_F32:.*]] = call @acosf(%[[IN1_F32]]) : (f32) -> f32 +// CHECK: %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32> +// CHECK: %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64> +// CHECK: %[[OUT0_F64:.*]] = call @acos(%[[IN0_F64]]) : (f64) -> f64 +// CHECK: %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64> +// CHECK: %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64> +// CHECK: %[[OUT1_F64:.*]] = call @acos(%[[IN1_F64]]) : (f64) -> f64 +// CHECK: %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64> +// CHECK: return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64> +// CHECK: } +func.func @acos_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) { + %float_result = math.acos %float : vector<2xf32> + %double_result = math.acos %double : vector<2xf64> + return %float_result, %double_result : vector<2xf32>, vector<2xf64> +} + // CHECK-LABEL: func @atan_caller // CHECK-SAME: %[[FLOAT:.*]]: f32 // CHECK-SAME: %[[DOUBLE:.*]]: f64 diff --git a/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir b/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir index fa9fff2dad66497b9d9d0efef367158841805a44..962ed7de584a2618bedbe45972d69834098cd49b 100644 --- a/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir +++ b/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir @@ -460,3 +460,33 @@ func.func @cast_f16_to_f32_write(%arg0: memref<16x16xf16>, %arg1: memref<16x16xf vector.transfer_write %cast, %arg3[%c0, %c0] {in_bounds = [true, true]} : vector<16x16xf32>, memref<16x16xf32> return } + +// ----- + +#map1 = affine_map<(d0, d1, d2) -> (d0, d2)> +#map2 = affine_map<(d0, d1, d2) -> (d2, d1)> +#map3 = affine_map<(d0, d1, d2) -> (d0, d1)> + +// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1) -> (d1, d0)> +// CHECK-LABEL: func @fold_transpose_into_transfer_read( +// CHECK-SAME: %[[ALLOC:.+]]: memref<64x128xf16> +// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index +// CHECK-DAG: %[[CST:.+]] = arith.constant 0.000000e+00 : f16 +// CHECK: %[[READ:.+]] = vector.transfer_read %[[ALLOC]][%[[C0]], %[[C0]]], %[[CST]] {in_bounds = [true, true], permutation_map = #[[$MAP]]} +// CHECK: %[[EXTF1:.+]] = arith.extf %[[READ]] +// CHECK-NOT: vector.transpose +// CHECK: %[[RESULT:.+]] = vector.contract +func.func @fold_transpose_into_transfer_read(%alloc: memref<64x128xf16>, %vector: vector<32x128xf16>, %alloc2: memref<32x64xf32>) { + %c0 = arith.constant 0 : index + %cst = arith.constant 0.000000e+00 : f16 + %init = arith.constant dense<0.000000e+00> : vector<32x64xf32> + %0 = vector.transfer_read %alloc[%c0, %c0], %cst {in_bounds = [true, true]} : memref<64x128xf16>, vector<64x128xf16> + %1 = arith.extf %0 : vector<64x128xf16> to vector<64x128xf32> + %2 = arith.extf %vector : vector<32x128xf16> to vector<32x128xf32> + %3 = vector.transpose %1, [1, 0] : vector<64x128xf32> to vector<128x64xf32> + %4 = vector.contract {indexing_maps = [#map1, #map2, #map3], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} %2, %3, %init : vector<32x128xf32>, vector<128x64xf32> into vector<32x64xf32> + vector.transfer_write %4, %alloc2[%c0, %c0] {in_bounds = [true, true]} : vector<32x64xf32>, memref<32x64xf32> + return +} + +// ----- diff --git a/mlir/test/Dialect/Arith/int-range-interface.mlir b/mlir/test/Dialect/Arith/int-range-interface.mlir index 4c5f4095225d3a4aa4659daaff50ac33d300ac57..02a9827d19d8f856433269d069488e21aaf0c52d 100644 --- a/mlir/test/Dialect/Arith/int-range-interface.mlir +++ b/mlir/test/Dialect/Arith/int-range-interface.mlir @@ -730,3 +730,29 @@ func.func @extui_uses_unsigned(%arg0 : i32) -> i1 { %4 = arith.andi %2, %3 : i1 func.return %4 : i1 } + +/// Catch a bug that caused a crash in getLoopBoundFromFold when +/// SparseConstantPropagation is loaded in the solver. + +// CHECK-LABEL: func.func @caller( +// CHECK-SAME: %[[VAL_0:.*]]: memref) { +// CHECK: call @callee(%[[VAL_0]]) : (memref) -> () +// CHECK: return +// CHECK: } +func.func @caller(%arg0: memref) { + call @callee(%arg0) : (memref) -> () + return +} + +// CHECK-LABEL: func.func private @callee( +// CHECK-SAME: %[[VAL_0:.*]]: memref) { +// CHECK: return +// CHECK: } +func.func private @callee(%arg0: memref) { + %c1 = arith.constant 1 : index + %c0 = arith.constant 0 : index + %0 = affine.load %arg0[0] : memref + scf.for %arg1 = %c0 to %0 step %c1 { + } + return +} diff --git a/mlir/test/Dialect/Complex/ops.mlir b/mlir/test/Dialect/Complex/ops.mlir index 1050ad0dcd5305a70fb7b15b8540a490db045c47..96f17b2898c834e4dc9ef91356fa703c18d12ca0 100644 --- a/mlir/test/Dialect/Complex/ops.mlir +++ b/mlir/test/Dialect/Complex/ops.mlir @@ -11,6 +11,9 @@ func.func @ops(%f: f32) { // CHECK: complex.constant [1.{{.*}} : f32, -1.{{.*}} : f32] : complex %cst_f32 = complex.constant [0.1 : f32, -1.0 : f32] : complex + // CHECK: complex.constant [true, false] : complex + %cst_i1 = complex.constant [1 : i1, 0 : i1] : complex + // CHECK: %[[C:.*]] = complex.create %[[F]], %[[F]] : complex %complex = complex.create %f, %f : complex diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir index 42589972ef2ba423204149d7d38bce2d69903aa3..594c3de91815aebcb34494002423eca552d45e8d 100644 --- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir +++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir @@ -34,6 +34,16 @@ func.func @ops(%arg0: i32, %arg1: f32, %vptrcmp = llvm.icmp "ne" %arg5, %arg5 : !llvm.vec<2 x ptr> %typecheck_vptrcmp = llvm.add %vptrcmp, %vptrcmp : vector<2 x i1> +// Integer overflow flags +// CHECK: {{.*}} = llvm.add %[[I32]], %[[I32]] overflow : i32 +// CHECK: {{.*}} = llvm.sub %[[I32]], %[[I32]] overflow : i32 +// CHECK: {{.*}} = llvm.mul %[[I32]], %[[I32]] overflow : i32 +// CHECK: {{.*}} = llvm.shl %[[I32]], %[[I32]] overflow : i32 + %add_flag = llvm.add %arg0, %arg0 overflow : i32 + %sub_flag = llvm.sub %arg0, %arg0 overflow : i32 + %mul_flag = llvm.mul %arg0, %arg0 overflow : i32 + %shl_flag = llvm.shl %arg0, %arg0 overflow : i32 + // Floating point binary operations. // // CHECK: {{.*}} = llvm.fadd %[[FLOAT]], %[[FLOAT]] : f32 diff --git a/mlir/test/Dialect/LLVMIR/type-consistency.mlir b/mlir/test/Dialect/LLVMIR/type-consistency.mlir index 1504a98e6f8cca443fd32fa240bb046d1267c1d1..3a1ab924ebdacb218a634e7e4d0973d7efdec188 100644 --- a/mlir/test/Dialect/LLVMIR/type-consistency.mlir +++ b/mlir/test/Dialect/LLVMIR/type-consistency.mlir @@ -151,6 +151,20 @@ llvm.func @index_to_struct(%arg: i32) { // ----- +// CHECK-LABEL: llvm.func @no_crash_on_negative_gep_index +llvm.func @no_crash_on_negative_gep_index() { + %0 = llvm.mlir.constant(1.000000e+00 : f16) : f16 + %1 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> + %2 = llvm.alloca %1 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK: llvm.getelementptr %[[ALLOCA]][-1] : (!llvm.ptr) -> !llvm.ptr, f32 + %3 = llvm.getelementptr %2[-1] : (!llvm.ptr) -> !llvm.ptr, f32 + llvm.store %0, %3 : f16, !llvm.ptr + llvm.return +} + +// ----- + // CHECK-LABEL: llvm.func @coalesced_store_ints // CHECK-SAME: %[[ARG:.*]]: i64 llvm.func @coalesced_store_ints(%arg: i64) { diff --git a/mlir/test/Dialect/Linalg/transform-op-match.mlir b/mlir/test/Dialect/Linalg/transform-op-match.mlir index 7d48b1f403b3bcb39bead2441facac73946bae77..fed3c007d9b6d8575bb51a72807741b88cc38cd2 100644 --- a/mlir/test/Dialect/Linalg/transform-op-match.mlir +++ b/mlir/test/Dialect/Linalg/transform-op-match.mlir @@ -43,6 +43,44 @@ module attributes {transform.with_named_sequence} { // ----- +func.func @by_operand_type() { + %c2 = arith.constant 2.0: f32 + %v = arith.constant 8: i32 + %r1 = math.fpowi %c2, %v : f32, i32 + // expected-remark @below {{matched op name}} + %r2 = arith.addf %c2, %c2 : f32 + // expected-remark @below {{matched op name}} + %r3 = arith.fptoui %r2 : f32 to i32 + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %match_name1 = transform.structured.match + ops{["arith.fptoui"]} filter_operand_types = [f32] in %arg1 : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %match_name1, "matched op name" : !transform.any_op + transform.test_consume_operand %match_name1 : !transform.any_op + + %match_name2 = transform.structured.match + ops{["arith.addf"]} filter_operand_types = [f32] in %arg1 : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %match_name2, "matched op name" : !transform.any_op + transform.test_consume_operand %match_name2 : !transform.any_op + + %no_match_name1 = transform.structured.match + ops{["arith.fptoui"]} filter_operand_types = [i32] in %arg1 : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %no_match_name1, "should not match" : !transform.any_op + transform.test_consume_operand %no_match_name1 : !transform.any_op + + %no_match_name2 = transform.structured.match + ops{["math.fpowi"]} filter_operand_types = [f32] in %arg1 : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %no_match_name2, "should not match" : !transform.any_op + transform.test_consume_operand %no_match_name2 : !transform.any_op + transform.yield + } +} + +// ----- + func.func @foo(%a: tensor<4x4xf32>, %b: tensor<4x4xf32>, %c: tensor<4x4xf32>) { %c0 = arith.constant 0.0 : f32 // expected-remark @below {{tileable}} diff --git a/mlir/test/Dialect/Linalg/transpose-conv2d.mlir b/mlir/test/Dialect/Linalg/transpose-conv2d.mlir index 4655a261d986b230d6c0271825b40ef88ace3489..409c36778ab7b5d69e91faad89bee6e36f6cc5ab 100644 --- a/mlir/test/Dialect/Linalg/transpose-conv2d.mlir +++ b/mlir/test/Dialect/Linalg/transpose-conv2d.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -transform-interpreter -verify-diagnostics | FileCheck %s +// RUN: mlir-opt %s -transform-interpreter | FileCheck %s // CHECK-LABEL: @conv_2d_nhwc_fhwc_f64 // CHECK-SAME: (%[[INPUT:.+]]: tensor<1x4x4x6xf64>, %[[FILTER:.+]]: tensor<8x2x2x6xf64>, %[[INIT:.+]]: tensor<1x2x2x8xf64>) -> tensor<1x2x2x8xf64> { diff --git a/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir b/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir index ab0c78a8ba76692a1ea06b105da024c84ea55e98..28b70043005940a03c0fcf72a4859738d8e321a7 100644 --- a/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir +++ b/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir @@ -1494,3 +1494,23 @@ func.func @extract_strided_metadata_of_cast_unranked( index, index, index, index } + + +// ----- +memref.global "private" @dynamicShmem : memref<0xf16,3> + +// CHECK-LABEL: func @zero_sized_memred +func.func @zero_sized_memred(%arg0: f32) -> (memref, index,index,index) { + %c0 = arith.constant 0 : index + %dynamicMem = memref.get_global @dynamicShmem : memref<0xf16, 3> + + // CHECK: %[[BASE:.*]] = memref.get_global @dynamicShmem : memref<0xf16, 3> + // CHECK: %[[CAST:.*]] = memref.reinterpret_cast %[[BASE]] to offset: [0], sizes: [], strides: [] : memref<0xf16, 3> to memref + // CHECK: return %[[CAST]] + + %base_buffer, %offset, %sizes, %strides = memref.extract_strided_metadata %dynamicMem : memref<0xf16, 3> -> memref, index, index, index + return %base_buffer, %offset, + %sizes, %strides : + memref, index, + index, index +} \ No newline at end of file diff --git a/mlir/test/Dialect/Mesh/canonicalization.mlir b/mlir/test/Dialect/Mesh/canonicalization.mlir index 5802d198d3681494d3b9f716c1c7df57ce4b72ea..baee9faa645c93a1601f13fbed74bd776dc6ab48 100644 --- a/mlir/test/Dialect/Mesh/canonicalization.mlir +++ b/mlir/test/Dialect/Mesh/canonicalization.mlir @@ -1,6 +1,6 @@ // RUN: mlir-opt --canonicalize %s | FileCheck %s -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) // CHECK-LABEL: func @all_reduce_empty_mesh_axes func.func @all_reduce_empty_mesh_axes( diff --git a/mlir/test/Dialect/Mesh/invalid.mlir b/mlir/test/Dialect/Mesh/invalid.mlir index 2999668f770baa7d33330419dc55715bf5e3a1cb..a26e3950186e95d4730eebf4dcde7806b03fc769 100644 --- a/mlir/test/Dialect/Mesh/invalid.mlir +++ b/mlir/test/Dialect/Mesh/invalid.mlir @@ -6,16 +6,16 @@ mesh.cluster @mesh0(rank = 0) // ----- // expected-error@+1 {{rank of dim_sizes is not expected to be larger than rank of cluster}} -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 3, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x3x4) // ----- -// expected-error@+1 {{dimension size of a mesh cluster is expected to be non-negative}} -mesh.cluster @mesh0(rank = 2, dim_sizes = [-1]) +// expected-error@+1 {{custom op 'mesh.cluster' Failed parsing dimension list. Did you mean an empty list? It must be denoted by "[]".}} +mesh.cluster @mesh0(rank = 2, dim_sizes = -1) // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @mesh_axis_duplicated_different_subarray( // expected-error@+1 {{mesh axis duplicated}} @@ -26,7 +26,7 @@ func.func @mesh_axis_duplicated_different_subarray( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @mesh_axis_duplicated_same_subarray( // expected-error@+1 {{mesh axis duplicated}} @@ -37,7 +37,7 @@ func.func @mesh_axis_duplicated_same_subarray( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @mesh_axis_duplicated_bewteen_split_and_partial( // expected-error@+1 {{mesh axis duplicated}} @@ -48,7 +48,7 @@ func.func @mesh_axis_duplicated_bewteen_split_and_partial( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @mesh_axis_negtive_in_split_part( // expected-error@+1 {{mesh axis is expected to be non-negative}} @@ -59,7 +59,7 @@ func.func @mesh_axis_negtive_in_split_part( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @mesh_axis_negtive_in_partial( // expected-error@+1 {{mesh axis is expected to be non-negative}} @@ -80,7 +80,7 @@ func.func @all_reduce_invalid_mesh_symbol( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @all_reduce_invalid_mesh_axis( %arg0 : tensor<4xf32>) -> tensor<4xf64> { @@ -92,7 +92,7 @@ func.func @all_reduce_invalid_mesh_axis( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @all_reduce_duplicate_mesh_axis( %arg0 : tensor<4xf32>) -> tensor<4xf64> { @@ -104,7 +104,7 @@ func.func @all_reduce_duplicate_mesh_axis( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @all_reduce_invalid_tensor_dimension_size( %arg0 : tensor<4xf32>) -> tensor<5xf64> { @@ -125,7 +125,7 @@ func.func @all_gather_invalid_mesh_symbol( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @all_gather_invalid_mesh_axis( %arg0 : tensor<4xf32>) -> tensor<4xf32> { @@ -137,7 +137,7 @@ func.func @all_gather_invalid_mesh_axis( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @all_reduce_duplicate_mesh_axis( %arg0 : tensor<4xf32>) -> tensor<4xf32> { @@ -149,7 +149,7 @@ func.func @all_reduce_duplicate_mesh_axis( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [1]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 1) func.func @all_gather_invalid_non_gather_axis_dimension_size( %arg0 : tensor<3x4xf32>) -> tensor<3x5xf32> { @@ -161,7 +161,7 @@ func.func @all_gather_invalid_non_gather_axis_dimension_size( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [1, 2]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 1x2) func.func @all_gather_invalid_gather_axis_dimension_size( %arg0 : tensor<3x4xf32>) -> tensor<3x5xf32> { @@ -173,7 +173,7 @@ func.func @all_gather_invalid_gather_axis_dimension_size( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [1]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 1) func.func @all_gather_invalid_gather_axis_dynamic_dimension( %arg0 : tensor) -> tensor<3xf32> { @@ -185,7 +185,7 @@ func.func @all_gather_invalid_gather_axis_dynamic_dimension( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [1]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 1) func.func @all_gather_invalid_gather_axis( %arg0 : tensor<3xf32>) -> tensor<3xf32> { @@ -197,7 +197,7 @@ func.func @all_gather_invalid_gather_axis( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [1]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 1) func.func @all_gather_invalid_negative_gather_axis( %arg0 : tensor<3xf32>) -> tensor<3xf32> { @@ -220,7 +220,7 @@ func.func @all_to_all_invalid_mesh_symbol( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [1]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 1) func.func @all_to_all_duplicate_mesh_axis( %arg0 : tensor<3x6xi8>) -> tensor<3x6xi8> { @@ -233,7 +233,7 @@ func.func @all_to_all_duplicate_mesh_axis( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [0, 1]) +mesh.cluster @mesh0(rank = 2, dim_sizes = ?x1) func.func @all_to_all_invalid_non_dynamic_result_dimension_induced_by_dynamic_device_group( %arg0 : tensor<3x6xi8>) -> tensor<3x6xi8> { @@ -246,7 +246,7 @@ func.func @all_to_all_invalid_non_dynamic_result_dimension_induced_by_dynamic_de // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [1, 1]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 1x1) func.func @all_to_all_invalid_non_dynamic_result_split_dimension_induced_by_dynamic_operand_dimension( %arg0 : tensor) -> tensor<3x?xi8> { @@ -259,7 +259,7 @@ func.func @all_to_all_invalid_non_dynamic_result_split_dimension_induced_by_dyna // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [1, 1]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 1x1) func.func @all_to_all_invalid_non_dynamic_result_concat_dimension_induced_by_dynamic_operand_dimension( %arg0 : tensor<3x?xi8>) -> tensor { @@ -272,7 +272,7 @@ func.func @all_to_all_invalid_non_dynamic_result_concat_dimension_induced_by_dyn // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [3]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 3) func.func @all_to_all_invalid_non_dynamic_result_concat_dimension_size( %arg0 : tensor<3x2xi8>) -> tensor<1x7xi8> { @@ -285,7 +285,7 @@ func.func @all_to_all_invalid_non_dynamic_result_concat_dimension_size( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [3]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 3) func.func @all_to_all_invalid_non_dynamic_result_split_dimension_size( %arg0 : tensor<3x2xi8>) -> tensor<2x6xi8> { @@ -298,7 +298,7 @@ func.func @all_to_all_invalid_non_dynamic_result_split_dimension_size( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [3]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 3) func.func @reduce_scatter_duplicate_mesh_axis( %arg0 : tensor) -> tensor { @@ -310,7 +310,7 @@ func.func @reduce_scatter_duplicate_mesh_axis( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [3]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 3) func.func @reduce_scatter_invalid_dynamic_dimension( %arg0 : tensor) -> tensor<2xf64> { @@ -322,7 +322,7 @@ func.func @reduce_scatter_invalid_dynamic_dimension( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [3]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 3) func.func @reduce_scatter_invalid_static_dimension_size( %arg0 : tensor<3xf32>) -> tensor<2xf64> { @@ -334,7 +334,7 @@ func.func @reduce_scatter_invalid_static_dimension_size( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [3]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 3) func.func @reduce_scatter_invalid_operand_static_dimension_size( %arg0 : tensor<4xf32>) -> tensor { diff --git a/mlir/test/Dialect/Mesh/ops.mlir b/mlir/test/Dialect/Mesh/ops.mlir index 5b264bc88dfc2a7850ad61a73dda5a4ccc60bd05..78ce276a7b33a331068f86faa67f2967345f20ac 100644 --- a/mlir/test/Dialect/Mesh/ops.mlir +++ b/mlir/test/Dialect/Mesh/ops.mlir @@ -1,18 +1,21 @@ // RUN: mlir-opt %s | mlir-opt | FileCheck %s // CHECK: mesh.cluster @mesh0 -mesh.cluster @mesh0(rank = 3, dim_sizes = [2, 2, 4]) +mesh.cluster @mesh0(rank = 3, dim_sizes = 2x2x4) -// CHECK: mesh.cluster @mesh1 -mesh.cluster @mesh1(rank = 2, dim_sizes = [4]) +// CHECK: mesh.cluster @mesh1(rank = 2, dim_sizes = 4) +mesh.cluster @mesh1(rank = 2, dim_sizes = 4) -// CHECK: mesh.cluster @mesh2 -mesh.cluster @mesh2(rank = 2, dim_sizes = [0, 4]) +// CHECK: mesh.cluster @mesh2(rank = 2, dim_sizes = ?x4) +mesh.cluster @mesh2(rank = 2, dim_sizes = ?x4) // CHECK: mesh.cluster @mesh3 mesh.cluster @mesh3(rank = 2) -mesh.cluster @mesh4(rank = 1, dim_sizes = [3]) +mesh.cluster @mesh4(rank = 1, dim_sizes = 3) + +// CHECK: mesh.cluster @mesh5(rank = 1) +mesh.cluster @mesh5(rank = 1, dim_sizes = []) // CHECK-LABEL: func @mesh_shard_encoding_fully_replicated func.func @mesh_shard_encoding_fully_replicated( diff --git a/mlir/test/Dialect/Mesh/sharding-propagation.mlir b/mlir/test/Dialect/Mesh/sharding-propagation.mlir index bda407b52bfd4f2f7163bd8d29aff5ab2c275a42..30bbd5c6619e8af1579477fa649bb8825f60f8f1 100644 --- a/mlir/test/Dialect/Mesh/sharding-propagation.mlir +++ b/mlir/test/Dialect/Mesh/sharding-propagation.mlir @@ -1,7 +1,7 @@ // RUN: mlir-opt -sharding-propagation %s | FileCheck %s mesh.cluster @mesh_1d(rank = 1) -mesh.cluster @mesh_2d(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh_2d(rank = 2, dim_sizes = 2x4) mesh.cluster @mesh_3d(rank = 3) // CHECK-LABEL: func.func @element_wise_empty_sharding_info diff --git a/mlir/test/Dialect/NVGPU/transform-pipeline-shared.mlir b/mlir/test/Dialect/NVGPU/transform-pipeline-shared.mlir index 42b072374261e0a5fc6cba7b27ac6f9d43d8342c..e959949babd9eb9598cfc93fe8bf0f8947c2b678 100644 --- a/mlir/test/Dialect/NVGPU/transform-pipeline-shared.mlir +++ b/mlir/test/Dialect/NVGPU/transform-pipeline-shared.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s --transform-interpreter --split-input-file --verify-diagnostics | FileCheck %s +// RUN: mlir-opt %s --transform-interpreter -canonicalize --split-input-file --verify-diagnostics | FileCheck %s func.func @simple_depth_2_unpeeled(%global: memref, %result: memref ) { %c0 = arith.constant 0 : index @@ -78,15 +78,19 @@ module attributes {transform.with_named_sequence} { // CHECK-LABEL: @async_depth_2_predicated // CHECK-SAME: %[[GLOBAL:.+]]: memref -func.func @async_depth_2_predicated(%global: memref) { +func.func @async_depth_2_predicated(%global: memref, %alloc_size: index) { %c0 = arith.constant 0 : index %c98 = arith.constant 98 : index %c100 = arith.constant 100 : index - %c200 = arith.constant 200 : index - // CHECK: %[[C4:.+]] = arith.constant 4 + // CHECK-DAG: %[[C4:.+]] = arith.constant 4 + // CHECK-DAG: %[[C90:.+]] = arith.constant 90 + // CHECK-DAG: %[[C96:.+]] = arith.constant 96 + // CHECK-DAG: %[[C8:.+]] = arith.constant 8 + // CHECK-DAG: %[[C2:.+]] = arith.constant 2 + // CHECK-DAG: %[[C0:.+]] = arith.constant 0 %c4 = arith.constant 4 : index // CHECK: %[[SHARED:.+]] = memref.alloc{{.*}} #gpu.address_space - %shared = memref.alloc(%c200) : memref> + %shared = memref.alloc(%alloc_size) : memref> %c0f = arith.constant 0.0 : f32 // CHECK: %[[TOKEN0:.+]] = nvgpu.device_async_copy // CHECK: %[[TOKEN1:.+]] = nvgpu.device_async_copy @@ -95,16 +99,11 @@ func.func @async_depth_2_predicated(%global: memref) { // CHECK-SAME: %[[ITER_ARG1:.+]] = %[[TOKEN1]] scf.for %i = %c0 to %c98 step %c4 { // Condition for the predication "select" below. - // CHECK: %[[C90:.+]] = arith.constant 90 // CHECK: %[[CMP0:.+]] = arith.cmpi slt, %[[I]], %[[C90]] // CHECK: nvgpu.device_async_wait %[[ITER_ARG0]] {numGroups = 1 - // Original "select" with updated induction variable. - // CHECK: %[[C96:.+]] = arith.constant 96 - // CHECK: %[[C8:.+]] = arith.constant 8 // CHECK: %[[I_PLUS_8:.+]] = arith.addi %[[I]], %[[C8]] // CHECK: %[[CMP1:.+]] = arith.cmpi slt, %[[I_PLUS_8]], %[[C96]] - // CHECK: %[[C2:.+]] = arith.constant 2 // CHECK: %[[SELECTED0:.+]] = arith.select %[[CMP1]], %[[C4]], %[[C2]] %c96 = arith.constant 96 : index %cond = arith.cmpi slt, %i, %c96 : index @@ -113,14 +112,11 @@ func.func @async_depth_2_predicated(%global: memref) { // Updated induction variables (two more) for the device_async_copy below. // These are generated repeatedly by the pipeliner. - // CHECK: %[[C8_2:.+]] = arith.constant 8 - // CHECK: %[[I_PLUS_8_2:.+]] = arith.addi %[[I]], %[[C8_2]] - // CHECK: %[[C8_3:.+]] = arith.constant 8 - // CHECK: %[[I_PLUS_8_3:.+]] = arith.addi %[[I]], %[[C8_3]] + // CHECK: %[[I_PLUS_8_2:.+]] = arith.addi %[[I]], %[[C8]] + // CHECK: %[[I_PLUS_8_3:.+]] = arith.addi %[[I]], %[[C8]] // The second "select" is generated by predication and selects 0 for // the two last iterations. - // CHECK: %[[C0:.+]] = arith.constant 0 // CHECK: %[[SELECTED1:.+]] = arith.select %[[CMP0]], %[[SELECTED0]], %[[C0]] // CHECK: %[[ASYNC_TOKEN:.+]] = nvgpu.device_async_copy %[[GLOBAL]][%[[I_PLUS_8_3]]], %[[SHARED]][%[[I_PLUS_8_2]]], 4, %[[SELECTED1]] %token = nvgpu.device_async_copy %global[%i], %shared[%i], 4, %read_size diff --git a/mlir/test/Dialect/OpenACC/ops.mlir b/mlir/test/Dialect/OpenACC/ops.mlir index c06e5a596923f9412e23d29342d0bd9fb7dd06d1..05b0450c7fb916bd048e38825eb292da23db0d12 100644 --- a/mlir/test/Dialect/OpenACC/ops.mlir +++ b/mlir/test/Dialect/OpenACC/ops.mlir @@ -974,7 +974,7 @@ func.func @testupdateop(%a: memref, %b: memref, %c: memref) -> () acc.update async(%idxValue: index) dataOperands(%0: memref) acc.update wait_devnum(%i64Value: i64) wait(%i32Value, %idxValue : i32, index) dataOperands(%0: memref) acc.update if(%ifCond) dataOperands(%0: memref) - acc.update dataOperands(%0: memref) attributes {acc.device_types = [#acc.device_type]} + acc.update dataOperands(%0: memref) attributes {acc.device_types = [#acc.device_type]} acc.update dataOperands(%0, %1, %2 : memref, memref, memref) acc.update dataOperands(%0, %1, %2 : memref, memref, memref) attributes {async} acc.update dataOperands(%0, %1, %2 : memref, memref, memref) attributes {wait} @@ -993,7 +993,7 @@ func.func @testupdateop(%a: memref, %b: memref, %c: memref) -> () // CHECK: acc.update async([[IDXVALUE]] : index) dataOperands(%{{.*}} : memref) // CHECK: acc.update wait_devnum([[I64VALUE]] : i64) wait([[I32VALUE]], [[IDXVALUE]] : i32, index) dataOperands(%{{.*}} : memref) // CHECK: acc.update if([[IFCOND]]) dataOperands(%{{.*}} : memref) -// CHECK: acc.update dataOperands(%{{.*}} : memref) attributes {acc.device_types = [#acc.device_type]} +// CHECK: acc.update dataOperands(%{{.*}} : memref) attributes {acc.device_types = [#acc.device_type]} // CHECK: acc.update dataOperands(%{{.*}}, %{{.*}}, %{{.*}} : memref, memref, memref) // CHECK: acc.update dataOperands(%{{.*}}, %{{.*}}, %{{.*}} : memref, memref, memref) attributes {async} // CHECK: acc.update dataOperands(%{{.*}}, %{{.*}}, %{{.*}} : memref, memref, memref) attributes {wait} diff --git a/mlir/test/Dialect/SCF/canonicalize.mlir b/mlir/test/Dialect/SCF/canonicalize.mlir index 9dbf8d5dab11ae68a1726e3d68266010f6871f1f..41e028028616a74459d2ef4c3038bd0341c25ede 100644 --- a/mlir/test/Dialect/SCF/canonicalize.mlir +++ b/mlir/test/Dialect/SCF/canonicalize.mlir @@ -773,56 +773,6 @@ func.func @remove_empty_parallel_loop(%lb: index, %ub: index, %s: index) { // ----- -func.func private @process(%0 : memref<128x128xf32>) -func.func private @process_tensor(%0 : tensor<128x128xf32>) -> memref<128x128xf32> - -// CHECK-LABEL: last_value -// CHECK-SAME: %[[T0:[0-9a-z]*]]: tensor<128x128xf32> -// CHECK-SAME: %[[T1:[0-9a-z]*]]: tensor<128x128xf32> -// CHECK-SAME: %[[T2:[0-9a-z]*]]: tensor<128x128xf32> -// CHECK-SAME: %[[M0:[0-9a-z]*]]: memref<128x128xf32> -func.func @last_value(%t0: tensor<128x128xf32>, %t1: tensor<128x128xf32>, - %t2: tensor<128x128xf32>, %m0: memref<128x128xf32>, - %lb : index, %ub : index, %step : index) - -> (tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32>) -{ - // CHECK-NEXT: %[[M1:.*]] = bufferization.to_memref %[[T1]] : memref<128x128xf32> - // CHECK-NEXT: %[[FOR_RES:.*]] = scf.for {{.*}} iter_args(%[[BBARG_T2:.*]] = %[[T2]]) -> (tensor<128x128xf32>) { - %0:3 = scf.for %arg0 = %lb to %ub step %step iter_args(%arg1 = %t0, %arg2 = %t1, %arg3 = %t2) - -> (tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32>) - { - %m1 = bufferization.to_memref %arg2 : memref<128x128xf32> - - // CHECK-NEXT: call @process(%[[M0]]) : (memref<128x128xf32>) -> () - func.call @process(%m0) : (memref<128x128xf32>) -> () - - // CHECK-NEXT: call @process(%[[M1]]) : (memref<128x128xf32>) -> () - func.call @process(%m1) : (memref<128x128xf32>) -> () - - // This does not hoist (fails the bbArg has at most a single check). - // CHECK-NEXT: %[[T:.*]] = func.call @process_tensor(%[[BBARG_T2]]) : (tensor<128x128xf32>) -> memref<128x128xf32> - // CHECK-NEXT: %[[YIELD_T:.*]] = bufferization.to_tensor %[[T:.*]] - %m2 = func.call @process_tensor(%arg3): (tensor<128x128xf32>) -> memref<128x128xf32> - %3 = bufferization.to_tensor %m2 : memref<128x128xf32> - - // All this stuff goes away, incrementally - %1 = bufferization.to_tensor %m0 : memref<128x128xf32> - %2 = bufferization.to_tensor %m1 : memref<128x128xf32> - - // CHECK-NEXT: scf.yield %[[YIELD_T]] : tensor<128x128xf32> - scf.yield %1, %2, %3 : tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32> - - // CHECK-NEXT: } - } - - // CHECK-NEXT: %[[R0:.*]] = bufferization.to_tensor %[[M0]] : memref<128x128xf32> - // CHECK-NEXT: %[[R1:.*]] = bufferization.to_tensor %[[M1]] : memref<128x128xf32> - // CHECK-NEXT: return %[[R0]], %[[R1]], %[[FOR_RES]] : tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32> - return %0#0, %0#1, %0#2 : tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32> -} - -// ----- - // CHECK-LABEL: fold_away_iter_with_no_use_and_yielded_input // CHECK-SAME: %[[A0:[0-9a-z]*]]: i32 func.func @fold_away_iter_with_no_use_and_yielded_input(%arg0 : i32, diff --git a/mlir/test/Dialect/SCF/loop-pipelining.mlir b/mlir/test/Dialect/SCF/loop-pipelining.mlir index 4cd686d2cdb86b69805a17589d4bef21d2a491f2..8a57ddccfee66594c54187e6d3cbae646be39103 100644 --- a/mlir/test/Dialect/SCF/loop-pipelining.mlir +++ b/mlir/test/Dialect/SCF/loop-pipelining.mlir @@ -723,3 +723,50 @@ func.func @stage_0_value_escape(%A: memref, %result: memref) { memref.store %r, %result[%c1] : memref return } + +// ----- + +// NOEPILOGUE-LABEL: dynamic_loop( +// NOEPILOGUE-SAME: %[[A:.*]]: memref, %[[R:.*]]: memref, %[[LB:.+]]: index, %[[UB:.+]]: index, %[[STEP:.+]]: index) { +// NOEPILOGUE-DAG: %[[C2:.+]] = arith.constant 2 : index +// NOEPILOGUE-DAG: %[[CSTF:.+]] = arith.constant 1.000000e+00 : f32 +// Prologue: +// NOEPILOGUE: %[[P_I0:.+]] = arith.cmpi slt, %[[LB]], %[[UB]] : index +// NOEPILOGUE: %[[L0:.+]] = scf.if %[[P_I0]] -> (f32) { +// NOEPILOGUE-NEXT: memref.load %[[A]][%[[LB]]] : memref +// NOEPILOGUE: %[[IV1:.+]] = arith.addi %[[LB]], %[[STEP]] : index +// NOEPILOGUE: %[[P_I1:.+]] = arith.cmpi slt, %[[IV1]], %[[UB]] : index +// NOEPILOGUE: %[[IV1_2:.+]] = arith.addi %[[LB]], %[[STEP]] : index +// NOEPILOGUE: %[[V0:.+]] = scf.if %[[P_I0]] -> (f32) { +// NOEPILOGUE-NEXT: arith.addf %[[L0]], %[[CSTF]] : f32 +// NOEPILOGUE: %[[L1:.+]] = scf.if %[[P_I1]] -> (f32) { +// NOEPILOGUE-NEXT: memref.load %[[A]][%[[IV1_2]]] : memref +// NOEPILOGUE: scf.for %[[IV2:.+]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[V1:.+]] = %[[V0]], %[[L2:.+]] = %[[L1]]) -> (f32, f32) { +// NOEPILOGUE-DAG: %[[S2:.+]] = arith.muli %[[STEP]], %[[C2]] : index +// NOEPILOGUE-DAG: %[[IT2:.+]] = arith.subi %[[UB]], %[[S2]] : index +// NOEPILOGUE-DAG: %[[P_I2:.+]] = arith.cmpi slt, %[[IV2]], %[[IT2]] : index +// NOEPILOGUE-DAG: %[[IT3:.+]] = arith.subi %[[UB]], %[[STEP]] : index +// NOEPILOGUE-DAG: %[[P_I3:.+]] = arith.cmpi slt, %[[IV2]], %[[IT3]] : index +// NOEPILOGUE: memref.store %[[V1]], %[[R]][%[[IV2]]] : memref +// NOEPILOGUE: %[[V2:.+]] = scf.if %[[P_I3]] -> (f32) { +// NOEPILOGUE: arith.addf %[[L2]], %[[CSTF]] : f32 +// NOEPILOGUE: %[[IT4:.+]] = arith.muli %[[STEP]], %[[C2]] : index +// NOEPILOGUE: %[[IV3:.+]] = arith.addi %[[IV2]], %[[IT4]] : index +// NOEPILOGUE: %[[L3:.+]] = scf.if %[[P_I2]] -> (f32) { +// NOEPILOGUE: memref.load %[[A]][%[[IV3]]] : memref +// NOEPILOGUE: scf.yield %[[V2]], %[[L3]] : f32, f32 + +// In case dynamic loop pipelining is off check that the transformation didn't +// apply. +// CHECK-LABEL: dynamic_loop( +// CHECK-NOT: memref.load +// CHECK: scf.for +func.func @dynamic_loop(%A: memref, %result: memref, %lb: index, %ub: index, %step: index) { + %cf = arith.constant 1.0 : f32 + scf.for %i0 = %lb to %ub step %step { + %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref + %A1_elem = arith.addf %A_elem, %cf { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : f32 + memref.store %A1_elem, %result[%i0] { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 0 } : memref + } { __test_pipelining_loop__ } + return +} diff --git a/mlir/test/Dialect/SPIRV/IR/bit-ops.mlir b/mlir/test/Dialect/SPIRV/IR/bit-ops.mlir index 82a2316f6c784fbeac3a449db682565dee78448b..f3f0ebf60f468e6cf961fc613c4f1ab1283fd0c9 100644 --- a/mlir/test/Dialect/SPIRV/IR/bit-ops.mlir +++ b/mlir/test/Dialect/SPIRV/IR/bit-ops.mlir @@ -149,14 +149,16 @@ func.func @bitwise_or_float(%arg0: f16, %arg1: f16) -> f16 { //===----------------------------------------------------------------------===// func.func @bitwise_xor_scalar(%arg: i32) -> i32 { + %c1 = spirv.Constant 1 : i32 // using constant to avoid folding // CHECK: spirv.BitwiseXor - %0 = spirv.BitwiseXor %arg, %arg : i32 + %0 = spirv.BitwiseXor %c1, %arg : i32 return %0 : i32 } func.func @bitwise_xor_vector(%arg: vector<4xi32>) -> vector<4xi32> { + %c1 = spirv.Constant dense<1> : vector<4xi32> // using constant to avoid folding // CHECK: spirv.BitwiseXor - %0 = spirv.BitwiseXor %arg, %arg : vector<4xi32> + %0 = spirv.BitwiseXor %c1, %arg : vector<4xi32> return %0 : vector<4xi32> } diff --git a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir index 867ddf3c8017336d1ab568581db531222252a42f..29bea91ce461d97cb7c8153a60d61f01223d0788 100644 --- a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir +++ b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir @@ -1139,6 +1139,372 @@ func.func @convert_logical_or_true_false_vector(%arg: vector<3xi1>) -> (vector<3 // ----- +//===----------------------------------------------------------------------===// +// spirv.LeftShiftLogical +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @lsl_x_0 +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @lsl_x_0(%arg0 : i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %c0 = spirv.Constant 0 : i32 + %cv0 = spirv.Constant dense<0> : vector<3xi32> + + %0 = spirv.ShiftLeftLogical %arg0, %c0 : i32, i32 + %1 = spirv.ShiftLeftLogical %arg1, %cv0 : vector<3xi32>, vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @lsl_shift_overflow +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @lsl_shift_overflow(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + // CHECK-DAG: %[[C32:.*]] = spirv.Constant 32 + // CHECK-DAG: %[[CV:.*]] = spirv.Constant dense<[6, 18, 128]> + %c32 = spirv.Constant 32 : i32 + %cv = spirv.Constant dense<[6, 18, 128]> : vector<3xi32> + + // CHECK: %0 = spirv.ShiftLeftLogical %[[ARG0]], %[[C32]] + // CHECK: %1 = spirv.ShiftLeftLogical %[[ARG1]], %[[CV]] + %0 = spirv.ShiftLeftLogical %arg0, %c32 : i32, i32 + %1 = spirv.ShiftLeftLogical %arg1, %cv : vector<3xi32>, vector<3xi32> + + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @const_fold_scalar_lsl +func.func @const_fold_scalar_lsl() -> i32 { + %c1 = spirv.Constant 65535 : i32 // 0x0000 ffff + %c2 = spirv.Constant 17 : i32 + + // CHECK: %[[RET:.*]] = spirv.Constant -131072 + // 0x0000 ffff << 17 -> 0xfffe 0000 + %0 = spirv.ShiftLeftLogical %c1, %c2 : i32, i32 + + // CHECK: return %[[RET]] + return %0 : i32 +} + +// CHECK-LABEL: @const_fold_vector_lsl +func.func @const_fold_vector_lsl() -> vector<3xi32> { + %c1 = spirv.Constant dense<[1, -1, 127]> : vector<3xi32> + %c2 = spirv.Constant dense<[31, 16, 13]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[-2147483648, -65536, 1040384]> + %0 = spirv.ShiftLeftLogical %c1, %c2 : vector<3xi32>, vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi32> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.RightShiftArithmetic +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @asr_x_0 +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @asr_x_0(%arg0 : i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %c0 = spirv.Constant 0 : i32 + %cv0 = spirv.Constant dense<0> : vector<3xi32> + + %0 = spirv.ShiftRightArithmetic %arg0, %c0 : i32, i32 + %1 = spirv.ShiftRightArithmetic %arg1, %cv0 : vector<3xi32>, vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @asr_shift_overflow +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @asr_shift_overflow(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + // CHECK-DAG: %[[C32:.*]] = spirv.Constant 32 + // CHECK-DAG: %[[CV:.*]] = spirv.Constant dense<[6, 18, 128]> + %c32 = spirv.Constant 32 : i32 + %cv = spirv.Constant dense<[6, 18, 128]> : vector<3xi32> + + // CHECK: %0 = spirv.ShiftRightArithmetic %[[ARG0]], %[[C32]] + // CHECK: %1 = spirv.ShiftRightArithmetic %[[ARG1]], %[[CV]] + %0 = spirv.ShiftRightArithmetic %arg0, %c32 : i32, i32 + %1 = spirv.ShiftRightArithmetic %arg1, %cv : vector<3xi32>, vector<3xi32> + + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @const_fold_scalar_asr +func.func @const_fold_scalar_asr() -> i32 { + %c1 = spirv.Constant -131072 : i32 // 0xfffe 0000 + %c2 = spirv.Constant 17 : i32 + // 0x0000 ffff ashr 17 -> 0xffff ffff + // CHECK: %[[RET:.*]] = spirv.Constant -1 + %0 = spirv.ShiftRightArithmetic %c1, %c2 : i32, i32 + + // CHECK: return %[[RET]] + return %0 : i32 +} + +// CHECK-LABEL: @const_fold_vector_asr +func.func @const_fold_vector_asr() -> vector<3xi32> { + %c1 = spirv.Constant dense<[-2147483648, 239847, 127]> : vector<3xi32> + %c2 = spirv.Constant dense<[31, 16, 13]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[-1, 3, 0]> + %0 = spirv.ShiftRightArithmetic %c1, %c2 : vector<3xi32>, vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi32> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.RightShiftLogical +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @lsr_x_0 +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @lsr_x_0(%arg0 : i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %c0 = spirv.Constant 0 : i32 + %cv0 = spirv.Constant dense<0> : vector<3xi32> + + %0 = spirv.ShiftRightLogical %arg0, %c0 : i32, i32 + %1 = spirv.ShiftRightLogical %arg1, %cv0 : vector<3xi32>, vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @lsr_shift_overflow +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @lsr_shift_overflow(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + // CHECK-DAG: %[[C32:.*]] = spirv.Constant 32 + // CHECK-DAG: %[[CV:.*]] = spirv.Constant dense<[6, 18, 128]> + %c32 = spirv.Constant 32 : i32 + %cv = spirv.Constant dense<[6, 18, 128]> : vector<3xi32> + + // CHECK: %0 = spirv.ShiftRightLogical %[[ARG0]], %[[C32]] + // CHECK: %1 = spirv.ShiftRightLogical %[[ARG1]], %[[CV]] + %0 = spirv.ShiftRightLogical %arg0, %c32 : i32, i32 + %1 = spirv.ShiftRightLogical %arg1, %cv : vector<3xi32>, vector<3xi32> + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @const_fold_scalar_lsr +func.func @const_fold_scalar_lsr() -> i32 { + %c1 = spirv.Constant -131072 : i32 // 0xfffe 0000 + %c2 = spirv.Constant 17 : i32 + + // 0x0000 ffff << 17 -> 0x0000 7fff + // CHECK: %[[RET:.*]] = spirv.Constant 32767 + %0 = spirv.ShiftRightLogical %c1, %c2 : i32, i32 + + // CHECK: return %[[RET]] + return %0 : i32 +} + +// CHECK-LABEL: @const_fold_vector_lsr +func.func @const_fold_vector_lsr() -> vector<3xi32> { + %c1 = spirv.Constant dense<[-2147483648, -1, -127]> : vector<3xi32> + %c2 = spirv.Constant dense<[31, 16, 13]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[1, 65535, 524287]> + %0 = spirv.ShiftRightLogical %c1, %c2 : vector<3xi32>, vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi32> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.BitwiseAnd +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @bitwise_and_x_x +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @bitwise_and_x_x(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %0 = spirv.BitwiseAnd %arg0, %arg0 : i32 + %1 = spirv.BitwiseAnd %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @bitwise_and_x_0 +func.func @bitwise_and_x_0(%arg0 : i32, %arg1 : vector<3xi32>) -> (i32, vector<3xi32>) { + // CHECK-DAG: %[[C0:.*]] = spirv.Constant 0 : i32 + // CHECK-DAG: %[[CV0:.*]] = spirv.Constant dense<0> : vector<3xi32> + %c0 = spirv.Constant 0 : i32 + %cv0 = spirv.Constant dense<0> : vector<3xi32> + + %0 = spirv.BitwiseAnd %arg0, %c0 : i32 + %1 = spirv.BitwiseAnd %arg1, %cv0 : vector<3xi32> + + // CHECK: return %[[C0]], %[[CV0]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @bitwise_and_x_n1 +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @bitwise_and_x_n1(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %cn1 = spirv.Constant -1 : i32 + %cvn1 = spirv.Constant dense<-1> : vector<3xi32> + %0 = spirv.BitwiseAnd %arg0, %cn1 : i32 + %1 = spirv.BitwiseAnd %arg1, %cvn1 : vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @const_fold_scalar_band +func.func @const_fold_scalar_band() -> i32 { + %c1 = spirv.Constant -268464129 : i32 // 0xefff 8fff + %c2 = spirv.Constant 268464128: i32 // 0x1000 7000 + + // 0xefff 8fff | 0x1000 7000 = 0xffff ffff = -1 + // CHECK: %[[C0:.*]] = spirv.Constant 0 + %0 = spirv.BitwiseAnd %c1, %c2 : i32 + + // CHECK: return %[[C0]] + return %0 : i32 +} + +// CHECK-LABEL: @const_fold_vector_band +func.func @const_fold_vector_band() -> vector<3xi32> { + %c1 = spirv.Constant dense<[42, -55, 127]> : vector<3xi32> + %c2 = spirv.Constant dense<[-3, -15, 28]> : vector<3xi32> + + // CHECK: %[[CV:.*]] = spirv.Constant dense<[40, -63, 28]> + %0 = spirv.BitwiseAnd %c1, %c2 : vector<3xi32> + + // CHECK: return %[[CV]] + return %0 : vector<3xi32> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.BitwiseOr +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @bitwise_or_x_x +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @bitwise_or_x_x(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %0 = spirv.BitwiseOr %arg0, %arg0 : i32 + %1 = spirv.BitwiseOr %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @bitwise_or_x_0 +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @bitwise_or_x_0(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %c1 = spirv.Constant 0 : i32 + %cv1 = spirv.Constant dense<0> : vector<3xi32> + %0 = spirv.BitwiseOr %arg0, %c1 : i32 + %1 = spirv.BitwiseOr %arg1, %cv1 : vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @bitwise_or_x_n1 +func.func @bitwise_or_x_n1(%arg0 : i32, %arg1 : vector<3xi32>) -> (i32, vector<3xi32>) { + // CHECK-DAG: %[[CN1:.*]] = spirv.Constant -1 : i32 + // CHECK-DAG: %[[CVN1:.*]] = spirv.Constant dense<-1> : vector<3xi32> + %cn1 = spirv.Constant -1 : i32 + %cvn1 = spirv.Constant dense<-1> : vector<3xi32> + %0 = spirv.BitwiseOr %arg0, %cn1 : i32 + %1 = spirv.BitwiseOr %arg1, %cvn1 : vector<3xi32> + + // CHECK: return %[[CN1]], %[[CVN1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @const_fold_scalar_bor +func.func @const_fold_scalar_bor() -> i32 { + %c1 = spirv.Constant -268464129 : i32 // 0xefff 8fff + %c2 = spirv.Constant 268464128: i32 // 0x1000 7000 + + // 0xefff 8fff | 0x1000 7000 = 0xffff ffff = -1 + // CHECK: %[[CN1:.*]] = spirv.Constant -1 + %0 = spirv.BitwiseOr %c1, %c2 : i32 + + // CHECK: return %[[CN1]] + return %0 : i32 +} + +// CHECK-LABEL: @const_fold_vector_bor +func.func @const_fold_vector_bor() -> vector<3xi32> { + %c1 = spirv.Constant dense<[42, -55, 127]> : vector<3xi32> + %c2 = spirv.Constant dense<[-3, -15, 28]> : vector<3xi32> + + // CHECK: %[[CV:.*]] = spirv.Constant dense<[-1, -7, 127]> + %0 = spirv.BitwiseOr %c1, %c2 : vector<3xi32> + + // CHECK: return %[[CV]] + return %0 : vector<3xi32> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.BitwiseXor +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @bitwise_xor_x_0 +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @bitwise_xor_x_0(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %c0 = spirv.Constant 0 : i32 + %cv0 = spirv.Constant dense<0> : vector<3xi32> + + %0 = spirv.BitwiseXor %arg0, %c0 : i32 + %1 = spirv.BitwiseXor %arg1, %cv0 : vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @bitwise_xor_x_x +func.func @bitwise_xor_x_x(%arg0 : i32, %arg1 : vector<3xi32>) -> (i32, vector<3xi32>) { + // CHECK-DAG: %[[C0:.*]] = spirv.Constant 0 + // CHECK-DAG: %[[CV0:.*]] = spirv.Constant dense<0> + %0 = spirv.BitwiseXor %arg0, %arg0 : i32 + %1 = spirv.BitwiseXor %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[C0]], %[[CV0]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @const_fold_scalar_bxor +func.func @const_fold_scalar_bxor() -> i32 { + %c1 = spirv.Constant 4294967295 : i32 // 2^32 - 1: 0xffff ffff + %c2 = spirv.Constant -2147483648 : i32 // -2^31 : 0x8000 0000 + + // 0x8000 0000 ^ 0xffff fffe = 0xefff ffff + // CHECK: %[[CBIG:.*]] = spirv.Constant 2147483647 + %0 = spirv.BitwiseXor %c1, %c2 : i32 + + // CHECK: return %[[CBIG]] + return %0 : i32 +} + +// CHECK-LABEL: @const_fold_vector_bxor +func.func @const_fold_vector_bxor() -> vector<3xi32> { + %c1 = spirv.Constant dense<[42, -55, 127]> : vector<3xi32> + %c2 = spirv.Constant dense<[-3, -15, 28]> : vector<3xi32> + + // CHECK: %[[CV:.*]] = spirv.Constant dense<[-41, 56, 99]> + %0 = spirv.BitwiseXor %c1, %c2 : vector<3xi32> + + // CHECK: return %[[CV]] + return %0 : vector<3xi32> +} + +// ----- + //===----------------------------------------------------------------------===// // spirv.mlir.selection //===----------------------------------------------------------------------===// diff --git a/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir b/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir index 0f99a0206e4cb85c9acee427e6173ee8661a2680..a3c1e76a3d09af37950a9c507b79b3ccb13bb22e 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir @@ -8,263 +8,230 @@ // CHECK-LABEL: func.func @conv2d_all_sparse_CSR( -// CHECK-SAME: %[[VAL_0:.*]]: tensor<8x8xi32, #sparse{{[0-9]*}}>, -// CHECK-SAME: %[[VAL_1:.*]]: tensor<3x3xi32>) -> tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK-SAME: %[[VAL_0:.*]]: tensor<8x8xi32, #sparse>, +// CHECK-SAME: %[[VAL_1:.*]]: tensor<3x3xi32>) -> tensor<6x6xi32, #sparse> { // CHECK-DAG: %[[VAL_2:.*]] = arith.constant true // CHECK-DAG: %[[VAL_3:.*]] = arith.constant -2 : index -// CHECK-DAG: %[[VAL_4:.*]] = arith.constant 4 : index +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant 2 : index // CHECK-DAG: %[[VAL_5:.*]] = arith.constant 8 : index // CHECK-DAG: %[[VAL_6:.*]] = arith.constant 3 : index // CHECK-DAG: %[[VAL_7:.*]] = arith.constant 1 : index -// CHECK-DAG: %[[VAL_8:.*]] = arith.constant 5 : index -// CHECK-DAG: %[[VAL_9:.*]] = arith.constant 2 : index -// CHECK-DAG: %[[VAL_10:.*]] = arith.constant 0 : index -// CHECK-DAG: %[[VAL_11:.*]] = arith.constant 0 : i32 -// CHECK-DAG: %[[VAL_12:.*]] = arith.constant false -// CHECK-DAG: %[[VAL_13:.*]] = tensor.empty() : tensor<6x6xi32, #sparse{{[0-9]*}}> -// CHECK-DAG: %[[VAL_14:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 0 : index} : tensor<8x8xi32, #sparse{{[0-9]*}}> to memref -// CHECK-DAG: %[[VAL_15:.*]] = sparse_tensor.coordinates %[[VAL_0]] {level = 0 : index} : tensor<8x8xi32, #sparse{{[0-9]*}}> to memref -// CHECK-DAG: %[[VAL_16:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 1 : index} : tensor<8x8xi32, #sparse{{[0-9]*}}> to memref -// CHECK-DAG: %[[VAL_17:.*]] = sparse_tensor.coordinates %[[VAL_0]] {level = 1 : index} : tensor<8x8xi32, #sparse{{[0-9]*}}> to memref -// CHECK-DAG: %[[VAL_18:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<8x8xi32, #sparse{{[0-9]*}}> to memref -// CHECK-DAG: %[[VAL_19:.*]] = memref.alloca() : memref<11xindex> -// CHECK-DAG: %[[VAL_20:.*]] = memref.alloca() : memref<5xindex> -// CHECK-DAG: %[[VAL_21:.*]] = memref.load %[[VAL_14]]{{\[}}%[[VAL_7]]] : memref -// CHECK-DAG: memref.store %[[VAL_7]], %[[VAL_20]]{{\[}}%[[VAL_10]]] : memref<5xindex> -// CHECK-DAG: memref.store %[[VAL_10]], %[[VAL_20]]{{\[}}%[[VAL_7]]] : memref<5xindex> -// CHECK-DAG: memref.store %[[VAL_10]], %[[VAL_20]]{{\[}}%[[VAL_9]]] : memref<5xindex> -// CHECK-DAG: memref.store %[[VAL_21]], %[[VAL_20]]{{\[}}%[[VAL_6]]] : memref<5xindex> -// CHECK: %[[VAL_22:.*]] = arith.cmpi ugt, %[[VAL_21]], %[[VAL_10]] : index -// CHECK: %[[VAL_23:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_10]]] : memref -// CHECK: %[[VAL_24:.*]] = arith.cmpi uge, %[[VAL_23]], %[[VAL_6]] : index -// CHECK: %[[VAL_25:.*]] = arith.andi %[[VAL_22]], %[[VAL_24]] : i1 -// CHECK: %[[VAL_26:.*]] = arith.addi %[[VAL_23]], %[[VAL_3]] : index -// CHECK: %[[VAL_27:.*]] = arith.select %[[VAL_25]], %[[VAL_26]], %[[VAL_10]] : index -// CHECK: %[[VAL_28:.*]]:3 = scf.while (%[[VAL_29:.*]] = %[[VAL_22]], %[[VAL_30:.*]] = %[[VAL_23]], %[[VAL_31:.*]] = %[[VAL_27]], %[[VAL_32:.*]] = %[[VAL_13]]) : (i1, index, index, tensor<6x6xi32, #sparse{{[0-9]*}}>) -> (index, index, tensor<6x6xi32, #sparse{{[0-9]*}}>) { -// CHECK: scf.condition(%[[VAL_29]]) %[[VAL_30]], %[[VAL_31]], %[[VAL_32]] : index, index, tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK-DAG: %[[VAL_8:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[VAL_9:.*]] = arith.constant 0 : i32 +// CHECK-DAG: %[[VAL_10:.*]] = arith.constant false +// CHECK-DAG: %[[VAL_11:.*]] = tensor.empty() : tensor<6x6xi32, #sparse> +// CHECK-DAG: %[[VAL_12:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 0 : index} : tensor<8x8xi32, #sparse> to memref +// CHECK-DAG: %[[VAL_13:.*]] = sparse_tensor.coordinates %[[VAL_0]] {level = 0 : index} : tensor<8x8xi32, #sparse> to memref +// CHECK-DAG: %[[VAL_14:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 1 : index} : tensor<8x8xi32, #sparse> to memref +// CHECK-DAG: %[[VAL_15:.*]] = sparse_tensor.coordinates %[[VAL_0]] {level = 1 : index} : tensor<8x8xi32, #sparse> to memref +// CHECK-DAG: %[[VAL_16:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<8x8xi32, #sparse> to memref +// CHECK-DAG: %[[VAL_17:.*]] = memref.alloca() : memref<9xindex> +// CHECK-DAG: %[[VAL_18:.*]] = memref.alloca() : memref<3xindex> +// CHECK-DAG: %[[VAL_19:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_7]]] : memref +// CHECK: memref.store %[[VAL_8]], %[[VAL_18]]{{\[}}%[[VAL_8]]] : memref<3xindex> +// CHECK: memref.store %[[VAL_19]], %[[VAL_18]]{{\[}}%[[VAL_7]]] : memref<3xindex> +// CHECK: %[[VAL_20:.*]] = arith.cmpi ugt, %[[VAL_19]], %[[VAL_8]] : index +// CHECK: %[[VAL_21:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_8]]] : memref +// CHECK: %[[VAL_22:.*]] = arith.cmpi uge, %[[VAL_21]], %[[VAL_6]] : index +// CHECK: %[[VAL_23:.*]] = arith.andi %[[VAL_20]], %[[VAL_22]] : i1 +// CHECK: %[[VAL_24:.*]] = arith.addi %[[VAL_21]], %[[VAL_3]] : index +// CHECK: %[[VAL_25:.*]] = arith.select %[[VAL_23]], %[[VAL_24]], %[[VAL_8]] : index +// CHECK: %[[VAL_26:.*]]:3 = scf.while (%[[VAL_27:.*]] = %[[VAL_20]], %[[VAL_28:.*]] = %[[VAL_21]], %[[VAL_29:.*]] = %[[VAL_25]], %[[VAL_30:.*]] = %[[VAL_11]]) : (i1, index, index, tensor<6x6xi32, #sparse>) -> (index, index, tensor<6x6xi32, #sparse>) { +// CHECK: scf.condition(%[[VAL_27]]) %[[VAL_28]], %[[VAL_29]], %[[VAL_30]] : index, index, tensor<6x6xi32, #sparse> // CHECK: } do { -// CHECK: ^bb0(%[[VAL_33:.*]]: index, %[[VAL_34:.*]]: index, %[[VAL_35:.*]]: tensor<6x6xi32, #sparse{{[0-9]*}}>): -// CHECK: %[[VAL_36:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_10]]] : memref<5xindex> -// CHECK: %[[VAL_37:.*]]:4 = scf.for %[[VAL_38:.*]] = %[[VAL_10]] to %[[VAL_36]] step %[[VAL_7]] iter_args(%[[VAL_39:.*]] = %[[VAL_12]], %[[VAL_40:.*]] = %[[VAL_5]], %[[VAL_41:.*]] = %[[VAL_10]], %[[VAL_42:.*]] = %[[VAL_10]]) -> (i1, index, index, index) { -// CHECK: %[[VAL_43:.*]] = arith.addi %[[VAL_38]], %[[VAL_9]] : index -// CHECK: %[[VAL_44:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_43]]] : memref<5xindex> -// CHECK: %[[VAL_45:.*]] = arith.addi %[[VAL_38]], %[[VAL_6]] : index -// CHECK: %[[VAL_46:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_45]]] : memref<5xindex> -// CHECK: %[[VAL_47:.*]] = arith.addi %[[VAL_38]], %[[VAL_4]] : index -// CHECK: memref.store %[[VAL_42]], %[[VAL_20]]{{\[}}%[[VAL_47]]] : memref<5xindex> -// CHECK: %[[VAL_48:.*]] = arith.addi %[[VAL_34]], %[[VAL_6]] : index -// CHECK: %[[VAL_49:.*]]:5 = scf.while (%[[VAL_50:.*]] = %[[VAL_44]], %[[VAL_51:.*]] = %[[VAL_39]], %[[VAL_52:.*]] = %[[VAL_40]], %[[VAL_53:.*]] = %[[VAL_41]], %[[VAL_54:.*]] = %[[VAL_42]]) : (index, i1, index, index, index) -> (index, i1, index, index, index) { -// CHECK: %[[VAL_55:.*]] = arith.cmpi ult, %[[VAL_50]], %[[VAL_46]] : index -// CHECK: %[[VAL_56:.*]] = scf.if %[[VAL_55]] -> (i1) { -// CHECK: %[[VAL_57:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_50]]] : memref -// CHECK: %[[VAL_58:.*]] = arith.cmpi ult, %[[VAL_57]], %[[VAL_48]] : index -// CHECK: scf.yield %[[VAL_58]] : i1 -// CHECK: } else { -// CHECK: scf.yield %[[VAL_12]] : i1 -// CHECK: } -// CHECK: scf.condition(%[[VAL_56]]) %[[VAL_50]], %[[VAL_51]], %[[VAL_52]], %[[VAL_53]], %[[VAL_54]] : index, i1, index, index, index -// CHECK: } do { -// CHECK: ^bb0(%[[VAL_59:.*]]: index, %[[VAL_60:.*]]: i1, %[[VAL_61:.*]]: index, %[[VAL_62:.*]]: index, %[[VAL_63:.*]]: index): -// CHECK: %[[VAL_64:.*]] = arith.addi %[[VAL_59]], %[[VAL_7]] : index -// CHECK: %[[VAL_65:.*]] = memref.load %[[VAL_16]]{{\[}}%[[VAL_59]]] : memref -// CHECK: %[[VAL_66:.*]] = memref.load %[[VAL_16]]{{\[}}%[[VAL_64]]] : memref -// CHECK: %[[VAL_67:.*]] = arith.cmpi ult, %[[VAL_65]], %[[VAL_66]] : index -// CHECK: %[[VAL_68:.*]] = arith.ori %[[VAL_67]], %[[VAL_60]] : i1 -// CHECK: %[[VAL_69:.*]] = scf.if %[[VAL_67]] -> (index) { -// CHECK: %[[VAL_70:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_65]]] : memref -// CHECK: %[[VAL_71:.*]] = arith.cmpi ult, %[[VAL_70]], %[[VAL_61]] : index -// CHECK: %[[VAL_72:.*]] = arith.select %[[VAL_71]], %[[VAL_70]], %[[VAL_61]] : index -// CHECK: scf.yield %[[VAL_72]] : index -// CHECK: } else { -// CHECK: scf.yield %[[VAL_61]] : index -// CHECK: } -// CHECK: %[[VAL_73:.*]] = arith.addi %[[VAL_62]], %[[VAL_9]] : index -// CHECK: memref.store %[[VAL_65]], %[[VAL_19]]{{\[}}%[[VAL_73]]] : memref<11xindex> -// CHECK: %[[VAL_74:.*]] = arith.addi %[[VAL_62]], %[[VAL_8]] : index -// CHECK: memref.store %[[VAL_66]], %[[VAL_19]]{{\[}}%[[VAL_74]]] : memref<11xindex> -// CHECK: %[[VAL_75:.*]] = arith.addi %[[VAL_62]], %[[VAL_7]] : index -// CHECK: %[[VAL_76:.*]] = arith.addi %[[VAL_63]], %[[VAL_7]] : index -// CHECK: scf.yield %[[VAL_64]], %[[VAL_68]], %[[VAL_69]], %[[VAL_75]], %[[VAL_76]] : index, i1, index, index, index +// CHECK: ^bb0(%[[VAL_31:.*]]: index, %[[VAL_32:.*]]: index, %[[VAL_33:.*]]: tensor<6x6xi32, #sparse>): +// CHECK: %[[VAL_34:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_8]]] : memref<3xindex> +// CHECK: %[[VAL_35:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_7]]] : memref<3xindex> +// CHECK: memref.store %[[VAL_8]], %[[VAL_18]]{{\[}}%[[VAL_4]]] : memref<3xindex> +// CHECK: %[[VAL_36:.*]] = arith.addi %[[VAL_32]], %[[VAL_6]] : index +// CHECK: %[[VAL_37:.*]]:5 = scf.while (%[[VAL_38:.*]] = %[[VAL_34]], %[[VAL_39:.*]] = %[[VAL_10]], %[[VAL_40:.*]] = %[[VAL_5]], %[[VAL_41:.*]] = %[[VAL_8]], %[[VAL_42:.*]] = %[[VAL_8]]) : (index, i1, index, index, index) -> (index, i1, index, index, index) { +// CHECK: %[[VAL_43:.*]] = arith.cmpi ult, %[[VAL_38]], %[[VAL_35]] : index +// CHECK: %[[VAL_44:.*]] = scf.if %[[VAL_43]] -> (i1) { +// CHECK: %[[VAL_45:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_38]]] : memref +// CHECK: %[[VAL_46:.*]] = arith.cmpi ult, %[[VAL_45]], %[[VAL_36]] : index +// CHECK: scf.yield %[[VAL_46]] : i1 +// CHECK: } else { +// CHECK: scf.yield %[[VAL_10]] : i1 +// CHECK: } +// CHECK: scf.condition(%[[VAL_44]]) %[[VAL_38]], %[[VAL_39]], %[[VAL_40]], %[[VAL_41]], %[[VAL_42]] : index, i1, index, index, index +// CHECK: } do { +// CHECK: ^bb0(%[[VAL_47:.*]]: index, %[[VAL_48:.*]]: i1, %[[VAL_49:.*]]: index, %[[VAL_50:.*]]: index, %[[VAL_51:.*]]: index): +// CHECK: %[[VAL_52:.*]] = arith.addi %[[VAL_47]], %[[VAL_7]] : index +// CHECK: %[[VAL_53:.*]] = memref.load %[[VAL_14]]{{\[}}%[[VAL_47]]] : memref +// CHECK: %[[VAL_54:.*]] = memref.load %[[VAL_14]]{{\[}}%[[VAL_52]]] : memref +// CHECK: %[[VAL_55:.*]] = arith.cmpi ult, %[[VAL_53]], %[[VAL_54]] : index +// CHECK: %[[VAL_56:.*]] = arith.ori %[[VAL_55]], %[[VAL_48]] : i1 +// CHECK: %[[VAL_57:.*]] = scf.if %[[VAL_55]] -> (index) { +// CHECK: %[[VAL_58:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_53]]] : memref +// CHECK: %[[VAL_59:.*]] = arith.cmpi ult, %[[VAL_58]], %[[VAL_49]] : index +// CHECK: %[[VAL_60:.*]] = arith.select %[[VAL_59]], %[[VAL_58]], %[[VAL_49]] : index +// CHECK: scf.yield %[[VAL_60]] : index +// CHECK: } else { +// CHECK: scf.yield %[[VAL_49]] : index // CHECK: } -// CHECK: scf.yield %[[VAL_77:.*]]#1, %[[VAL_77]]#2, %[[VAL_77]]#3, %[[VAL_77]]#4 : i1, index, index, index +// CHECK: memref.store %[[VAL_53]], %[[VAL_17]]{{\[}}%[[VAL_50]]] : memref<9xindex> +// CHECK: %[[VAL_61:.*]] = arith.addi %[[VAL_50]], %[[VAL_6]] : index +// CHECK: memref.store %[[VAL_54]], %[[VAL_17]]{{\[}}%[[VAL_61]]] : memref<9xindex> +// CHECK: %[[VAL_62:.*]] = arith.addi %[[VAL_50]], %[[VAL_7]] : index +// CHECK: %[[VAL_63:.*]] = arith.addi %[[VAL_51]], %[[VAL_7]] : index +// CHECK: scf.yield %[[VAL_52]], %[[VAL_56]], %[[VAL_57]], %[[VAL_62]], %[[VAL_63]] : index, i1, index, index, index // CHECK: } -// CHECK: memref.store %[[VAL_78:.*]]#2, %[[VAL_19]]{{\[}}%[[VAL_10]]] : memref<11xindex> -// CHECK: memref.store %[[VAL_10]], %[[VAL_19]]{{\[}}%[[VAL_7]]] : memref<11xindex> -// CHECK: %[[VAL_79:.*]] = arith.cmpi uge, %[[VAL_78]]#1, %[[VAL_6]] : index -// CHECK: %[[VAL_80:.*]] = arith.andi %[[VAL_78]]#0, %[[VAL_79]] : i1 -// CHECK: %[[VAL_81:.*]] = arith.addi %[[VAL_78]]#1, %[[VAL_3]] : index -// CHECK: %[[VAL_82:.*]] = arith.select %[[VAL_80]], %[[VAL_81]], %[[VAL_10]] : index -// CHECK: %[[VAL_83:.*]]:3 = scf.while (%[[VAL_84:.*]] = %[[VAL_78]]#0, %[[VAL_85:.*]] = %[[VAL_78]]#1, %[[VAL_86:.*]] = %[[VAL_82]], %[[VAL_87:.*]] = %[[VAL_35]]) : (i1, index, index, tensor<6x6xi32, #sparse{{[0-9]*}}>) -> (index, index, tensor<6x6xi32, #sparse{{[0-9]*}}>) { -// CHECK: scf.condition(%[[VAL_84]]) %[[VAL_85]], %[[VAL_86]], %[[VAL_87]] : index, index, tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_64:.*]] = arith.cmpi uge, %[[VAL_65:.*]]#2, %[[VAL_6]] : index +// CHECK: %[[VAL_66:.*]] = arith.andi %[[VAL_65]]#1, %[[VAL_64]] : i1 +// CHECK: %[[VAL_67:.*]] = arith.addi %[[VAL_65]]#2, %[[VAL_3]] : index +// CHECK: %[[VAL_68:.*]] = arith.select %[[VAL_66]], %[[VAL_67]], %[[VAL_8]] : index +// CHECK: %[[VAL_69:.*]]:3 = scf.while (%[[VAL_70:.*]] = %[[VAL_65]]#1, %[[VAL_71:.*]] = %[[VAL_65]]#2, %[[VAL_72:.*]] = %[[VAL_68]], %[[VAL_73:.*]] = %[[VAL_33]]) : (i1, index, index, tensor<6x6xi32, #sparse>) -> (index, index, tensor<6x6xi32, #sparse>) { +// CHECK: scf.condition(%[[VAL_70]]) %[[VAL_71]], %[[VAL_72]], %[[VAL_73]] : index, index, tensor<6x6xi32, #sparse> // CHECK: } do { -// CHECK: ^bb0(%[[VAL_88:.*]]: index, %[[VAL_89:.*]]: index, %[[VAL_90:.*]]: tensor<6x6xi32, #sparse{{[0-9]*}}>): -// CHECK: %[[VAL_91:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_7]]] : memref<5xindex> -// CHECK: %[[VAL_92:.*]] = arith.addi %[[VAL_91]], %[[VAL_9]] : index -// CHECK: %[[VAL_93:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_92]]] : memref<5xindex> -// CHECK: %[[VAL_94:.*]] = arith.addi %[[VAL_91]], %[[VAL_6]] : index -// CHECK: %[[VAL_95:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_94]]] : memref<5xindex> -// CHECK: %[[VAL_96:.*]] = arith.addi %[[VAL_34]], %[[VAL_6]] : index -// CHECK: %[[VAL_97:.*]]:3 = scf.while (%[[VAL_98:.*]] = %[[VAL_93]], %[[VAL_99:.*]] = %[[VAL_11]], %[[VAL_100:.*]] = %[[VAL_12]]) : (index, i32, i1) -> (index, i32, i1) { -// CHECK: %[[VAL_101:.*]] = arith.cmpi ult, %[[VAL_98]], %[[VAL_95]] : index -// CHECK: %[[VAL_102:.*]] = scf.if %[[VAL_101]] -> (i1) { -// CHECK: %[[VAL_103:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_98]]] : memref -// CHECK: %[[VAL_104:.*]] = arith.cmpi ult, %[[VAL_103]], %[[VAL_96]] : index -// CHECK: scf.yield %[[VAL_104]] : i1 +// CHECK: ^bb0(%[[VAL_74:.*]]: index, %[[VAL_75:.*]]: index, %[[VAL_76:.*]]: tensor<6x6xi32, #sparse>): +// CHECK: %[[VAL_77:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_8]]] : memref<3xindex> +// CHECK: %[[VAL_78:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_7]]] : memref<3xindex> +// CHECK: %[[VAL_79:.*]]:3 = scf.while (%[[VAL_80:.*]] = %[[VAL_77]], %[[VAL_81:.*]] = %[[VAL_9]], %[[VAL_82:.*]] = %[[VAL_10]]) : (index, i32, i1) -> (index, i32, i1) { +// CHECK: %[[VAL_83:.*]] = arith.cmpi ult, %[[VAL_80]], %[[VAL_78]] : index +// CHECK: %[[VAL_84:.*]] = scf.if %[[VAL_83]] -> (i1) { +// CHECK: %[[VAL_85:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_80]]] : memref +// CHECK: %[[VAL_86:.*]] = arith.cmpi ult, %[[VAL_85]], %[[VAL_36]] : index +// CHECK: scf.yield %[[VAL_86]] : i1 // CHECK: } else { -// CHECK: scf.yield %[[VAL_12]] : i1 +// CHECK: scf.yield %[[VAL_10]] : i1 // CHECK: } -// CHECK: scf.condition(%[[VAL_102]]) %[[VAL_98]], %[[VAL_99]], %[[VAL_100]] : index, i32, i1 +// CHECK: scf.condition(%[[VAL_84]]) %[[VAL_80]], %[[VAL_81]], %[[VAL_82]] : index, i32, i1 // CHECK: } do { -// CHECK: ^bb0(%[[VAL_105:.*]]: index, %[[VAL_106:.*]]: i32, %[[VAL_107:.*]]: i1): -// CHECK: %[[VAL_108:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_105]]] : memref -// CHECK: %[[VAL_109:.*]] = arith.subi %[[VAL_108]], %[[VAL_34]] : index -// CHECK: %[[VAL_110:.*]] = memref.load %[[VAL_19]]{{\[}}%[[VAL_7]]] : memref<11xindex> -// CHECK: %[[VAL_111:.*]] = arith.addi %[[VAL_110]], %[[VAL_9]] : index -// CHECK: %[[VAL_112:.*]] = memref.load %[[VAL_19]]{{\[}}%[[VAL_111]]] : memref<11xindex> -// CHECK: %[[VAL_113:.*]] = arith.addi %[[VAL_110]], %[[VAL_8]] : index -// CHECK: %[[VAL_114:.*]] = memref.load %[[VAL_19]]{{\[}}%[[VAL_113]]] : memref<11xindex> -// CHECK: %[[VAL_115:.*]] = arith.addi %[[VAL_89]], %[[VAL_6]] : index -// CHECK: %[[VAL_116:.*]]:2 = scf.while (%[[VAL_117:.*]] = %[[VAL_112]], %[[VAL_118:.*]] = %[[VAL_106]]) : (index, i32) -> (index, i32) { -// CHECK: %[[VAL_119:.*]] = arith.cmpi ult, %[[VAL_117]], %[[VAL_114]] : index -// CHECK: %[[VAL_120:.*]] = scf.if %[[VAL_119]] -> (i1) { -// CHECK: %[[VAL_121:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_117]]] : memref -// CHECK: %[[VAL_122:.*]] = arith.cmpi ult, %[[VAL_121]], %[[VAL_115]] : index -// CHECK: scf.yield %[[VAL_122]] : i1 +// CHECK: ^bb0(%[[VAL_87:.*]]: index, %[[VAL_88:.*]]: i32, %[[VAL_89:.*]]: i1): +// CHECK: %[[VAL_90:.*]] = arith.subi %[[VAL_87]], %[[VAL_77]] : index +// CHECK: %[[VAL_91:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_87]]] : memref +// CHECK: %[[VAL_92:.*]] = arith.subi %[[VAL_91]], %[[VAL_32]] : index +// CHECK: %[[VAL_93:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_90]]] : memref<9xindex> +// CHECK: %[[VAL_94:.*]] = arith.addi %[[VAL_90]], %[[VAL_6]] : index +// CHECK: %[[VAL_95:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_94]]] : memref<9xindex> +// CHECK: %[[VAL_96:.*]] = arith.addi %[[VAL_75]], %[[VAL_6]] : index +// CHECK: %[[VAL_97:.*]]:2 = scf.while (%[[VAL_98:.*]] = %[[VAL_93]], %[[VAL_99:.*]] = %[[VAL_88]]) : (index, i32) -> (index, i32) { +// CHECK: %[[VAL_100:.*]] = arith.cmpi ult, %[[VAL_98]], %[[VAL_95]] : index +// CHECK: %[[VAL_101:.*]] = scf.if %[[VAL_100]] -> (i1) { +// CHECK: %[[VAL_102:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_98]]] : memref +// CHECK: %[[VAL_103:.*]] = arith.cmpi ult, %[[VAL_102]], %[[VAL_96]] : index +// CHECK: scf.yield %[[VAL_103]] : i1 // CHECK: } else { -// CHECK: scf.yield %[[VAL_12]] : i1 +// CHECK: scf.yield %[[VAL_10]] : i1 // CHECK: } -// CHECK: scf.condition(%[[VAL_120]]) %[[VAL_117]], %[[VAL_118]] : index, i32 +// CHECK: scf.condition(%[[VAL_101]]) %[[VAL_98]], %[[VAL_99]] : index, i32 // CHECK: } do { -// CHECK: ^bb0(%[[VAL_123:.*]]: index, %[[VAL_124:.*]]: i32): -// CHECK: %[[VAL_125:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_123]]] : memref -// CHECK: %[[VAL_126:.*]] = arith.subi %[[VAL_125]], %[[VAL_89]] : index -// CHECK: %[[VAL_127:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_123]]] : memref -// CHECK: %[[VAL_128:.*]] = tensor.extract %[[VAL_1]]{{\[}}%[[VAL_109]], %[[VAL_126]]] : tensor<3x3xi32> -// CHECK: %[[VAL_129:.*]] = arith.muli %[[VAL_127]], %[[VAL_128]] : i32 -// CHECK: %[[VAL_130:.*]] = arith.addi %[[VAL_124]], %[[VAL_129]] : i32 -// CHECK: %[[VAL_131:.*]] = arith.addi %[[VAL_123]], %[[VAL_7]] : index -// CHECK: scf.yield %[[VAL_131]], %[[VAL_130]] : index, i32 +// CHECK: ^bb0(%[[VAL_104:.*]]: index, %[[VAL_105:.*]]: i32): +// CHECK: %[[VAL_106:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_104]]] : memref +// CHECK: %[[VAL_107:.*]] = arith.subi %[[VAL_106]], %[[VAL_75]] : index +// CHECK: %[[VAL_108:.*]] = memref.load %[[VAL_16]]{{\[}}%[[VAL_104]]] : memref +// CHECK: %[[VAL_109:.*]] = tensor.extract %[[VAL_1]]{{\[}}%[[VAL_92]], %[[VAL_107]]] : tensor<3x3xi32> +// CHECK: %[[VAL_110:.*]] = arith.muli %[[VAL_108]], %[[VAL_109]] : i32 +// CHECK: %[[VAL_111:.*]] = arith.addi %[[VAL_105]], %[[VAL_110]] : i32 +// CHECK: %[[VAL_112:.*]] = arith.addi %[[VAL_104]], %[[VAL_7]] : index +// CHECK: scf.yield %[[VAL_112]], %[[VAL_111]] : index, i32 // CHECK: } -// CHECK: %[[VAL_132:.*]] = arith.addi %[[VAL_105]], %[[VAL_7]] : index -// CHECK: %[[VAL_133:.*]] = arith.addi %[[VAL_110]], %[[VAL_7]] : index -// CHECK: memref.store %[[VAL_133]], %[[VAL_19]]{{\[}}%[[VAL_7]]] : memref<11xindex> -// CHECK: scf.yield %[[VAL_132]], %[[VAL_134:.*]]#1, %[[VAL_2]] : index, i32, i1 +// CHECK: %[[VAL_113:.*]] = arith.addi %[[VAL_87]], %[[VAL_7]] : index +// CHECK: scf.yield %[[VAL_113]], %[[VAL_114:.*]]#1, %[[VAL_2]] : index, i32, i1 // CHECK: } -// CHECK: %[[VAL_135:.*]] = scf.if %[[VAL_136:.*]]#2 -> (tensor<6x6xi32, #sparse{{[0-9]*}}>) { -// CHECK: %[[VAL_137:.*]] = sparse_tensor.insert %[[VAL_136]]#1 into %[[VAL_90]]{{\[}}%[[VAL_34]], %[[VAL_89]]] : tensor<6x6xi32, #sparse{{[0-9]*}}> -// CHECK: scf.yield %[[VAL_137]] : tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_115:.*]] = scf.if %[[VAL_116:.*]]#2 -> (tensor<6x6xi32, #sparse>) { +// CHECK: %[[VAL_117:.*]] = sparse_tensor.insert %[[VAL_116]]#1 into %[[VAL_76]]{{\[}}%[[VAL_32]], %[[VAL_75]]] : tensor<6x6xi32, #sparse> +// CHECK: scf.yield %[[VAL_117]] : tensor<6x6xi32, #sparse> // CHECK: } else { -// CHECK: scf.yield %[[VAL_90]] : tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK: scf.yield %[[VAL_76]] : tensor<6x6xi32, #sparse> // CHECK: } -// CHECK: memref.store %[[VAL_10]], %[[VAL_20]]{{\[}}%[[VAL_7]]] : memref<5xindex> -// CHECK: memref.store %[[VAL_10]], %[[VAL_19]]{{\[}}%[[VAL_7]]] : memref<11xindex> -// CHECK: %[[VAL_138:.*]] = arith.cmpi ugt, %[[VAL_88]], %[[VAL_89]] : index -// CHECK: %[[VAL_139:.*]]:3 = scf.if %[[VAL_138]] -> (index, i1, index) { -// CHECK: %[[VAL_140:.*]] = arith.addi %[[VAL_89]], %[[VAL_7]] : index -// CHECK: scf.yield %[[VAL_88]], %[[VAL_2]], %[[VAL_140]] : index, i1, index +// CHECK: %[[VAL_118:.*]] = arith.cmpi ugt, %[[VAL_74]], %[[VAL_75]] : index +// CHECK: %[[VAL_119:.*]]:3 = scf.if %[[VAL_118]] -> (index, i1, index) { +// CHECK: %[[VAL_120:.*]] = arith.addi %[[VAL_75]], %[[VAL_7]] : index +// CHECK: scf.yield %[[VAL_74]], %[[VAL_2]], %[[VAL_120]] : index, i1, index // CHECK: } else { -// CHECK: %[[VAL_141:.*]] = memref.load %[[VAL_19]]{{\[}}%[[VAL_10]]] : memref<11xindex> -// CHECK: %[[VAL_142:.*]]:2 = scf.for %[[VAL_143:.*]] = %[[VAL_10]] to %[[VAL_141]] step %[[VAL_7]] iter_args(%[[VAL_144:.*]] = %[[VAL_5]], %[[VAL_145:.*]] = %[[VAL_12]]) -> (index, i1) { -// CHECK: %[[VAL_146:.*]] = arith.addi %[[VAL_143]], %[[VAL_9]] : index -// CHECK: %[[VAL_147:.*]] = memref.load %[[VAL_19]]{{\[}}%[[VAL_146]]] : memref<11xindex> -// CHECK: %[[VAL_148:.*]] = arith.addi %[[VAL_143]], %[[VAL_8]] : index -// CHECK: %[[VAL_149:.*]] = memref.load %[[VAL_19]]{{\[}}%[[VAL_148]]] : memref<11xindex> -// CHECK: %[[VAL_150:.*]] = arith.cmpi ult, %[[VAL_147]], %[[VAL_149]] : index -// CHECK: %[[VAL_151:.*]] = scf.if %[[VAL_150]] -> (index) { -// CHECK: %[[VAL_152:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_147]]] : memref -// CHECK: %[[VAL_153:.*]] = arith.cmpi eq, %[[VAL_152]], %[[VAL_88]] : index -// CHECK: %[[VAL_154:.*]] = scf.if %[[VAL_153]] -> (index) { -// CHECK: %[[VAL_155:.*]] = arith.addi %[[VAL_147]], %[[VAL_7]] : index -// CHECK: memref.store %[[VAL_155]], %[[VAL_19]]{{\[}}%[[VAL_146]]] : memref<11xindex> -// CHECK: scf.yield %[[VAL_155]] : index +// CHECK: %[[VAL_121:.*]]:2 = scf.for %[[VAL_122:.*]] = %[[VAL_8]] to %[[VAL_65]]#3 step %[[VAL_7]] iter_args(%[[VAL_123:.*]] = %[[VAL_5]], %[[VAL_124:.*]] = %[[VAL_10]]) -> (index, i1) { +// CHECK: %[[VAL_125:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_122]]] : memref<9xindex> +// CHECK: %[[VAL_126:.*]] = arith.addi %[[VAL_122]], %[[VAL_6]] : index +// CHECK: %[[VAL_127:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_126]]] : memref<9xindex> +// CHECK: %[[VAL_128:.*]] = arith.cmpi ult, %[[VAL_125]], %[[VAL_127]] : index +// CHECK: %[[VAL_129:.*]] = scf.if %[[VAL_128]] -> (index) { +// CHECK: %[[VAL_130:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_125]]] : memref +// CHECK: %[[VAL_131:.*]] = arith.cmpi eq, %[[VAL_130]], %[[VAL_74]] : index +// CHECK: %[[VAL_132:.*]] = scf.if %[[VAL_131]] -> (index) { +// CHECK: %[[VAL_133:.*]] = arith.addi %[[VAL_125]], %[[VAL_7]] : index +// CHECK: memref.store %[[VAL_133]], %[[VAL_17]]{{\[}}%[[VAL_122]]] : memref<9xindex> +// CHECK: scf.yield %[[VAL_133]] : index // CHECK: } else { -// CHECK: scf.yield %[[VAL_147]] : index +// CHECK: scf.yield %[[VAL_125]] : index // CHECK: } -// CHECK: scf.yield %[[VAL_154]] : index +// CHECK: scf.yield %[[VAL_132]] : index // CHECK: } else { -// CHECK: scf.yield %[[VAL_147]] : index +// CHECK: scf.yield %[[VAL_125]] : index // CHECK: } -// CHECK: %[[VAL_156:.*]] = arith.cmpi ult, %[[VAL_151]], %[[VAL_149]] : index -// CHECK: %[[VAL_157:.*]] = scf.if %[[VAL_156]] -> (index) { -// CHECK: %[[VAL_158:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_151]]] : memref -// CHECK: scf.yield %[[VAL_158]] : index +// CHECK: %[[VAL_134:.*]] = arith.cmpi ult, %[[VAL_129]], %[[VAL_127]] : index +// CHECK: %[[VAL_135:.*]] = scf.if %[[VAL_134]] -> (index) { +// CHECK: %[[VAL_136:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_129]]] : memref +// CHECK: scf.yield %[[VAL_136]] : index // CHECK: } else { -// CHECK: scf.yield %[[VAL_144]] : index +// CHECK: scf.yield %[[VAL_123]] : index // CHECK: } -// CHECK: %[[VAL_159:.*]] = arith.ori %[[VAL_156]], %[[VAL_145]] : i1 -// CHECK: %[[VAL_160:.*]] = arith.cmpi ult, %[[VAL_157]], %[[VAL_144]] : index -// CHECK: %[[VAL_161:.*]] = arith.select %[[VAL_160]], %[[VAL_157]], %[[VAL_144]] : index -// CHECK: scf.yield %[[VAL_161]], %[[VAL_159]] : index, i1 +// CHECK: %[[VAL_137:.*]] = arith.ori %[[VAL_134]], %[[VAL_124]] : i1 +// CHECK: %[[VAL_138:.*]] = arith.cmpi ult, %[[VAL_135]], %[[VAL_123]] : index +// CHECK: %[[VAL_139:.*]] = arith.select %[[VAL_138]], %[[VAL_135]], %[[VAL_123]] : index +// CHECK: scf.yield %[[VAL_139]], %[[VAL_137]] : index, i1 // CHECK: } -// CHECK: %[[VAL_162:.*]] = arith.addi %[[VAL_163:.*]]#0, %[[VAL_7]] : index -// CHECK: %[[VAL_164:.*]] = arith.addi %[[VAL_163]]#0, %[[VAL_3]] : index -// CHECK: %[[VAL_165:.*]] = arith.cmpi uge, %[[VAL_162]], %[[VAL_6]] : index -// CHECK: %[[VAL_166:.*]] = arith.select %[[VAL_165]], %[[VAL_164]], %[[VAL_10]] : index -// CHECK: scf.yield %[[VAL_163]]#0, %[[VAL_163]]#1, %[[VAL_166]] : index, i1, index +// CHECK: %[[VAL_140:.*]] = arith.addi %[[VAL_141:.*]]#0, %[[VAL_7]] : index +// CHECK: %[[VAL_142:.*]] = arith.addi %[[VAL_141]]#0, %[[VAL_3]] : index +// CHECK: %[[VAL_143:.*]] = arith.cmpi uge, %[[VAL_140]], %[[VAL_6]] : index +// CHECK: %[[VAL_144:.*]] = arith.select %[[VAL_143]], %[[VAL_142]], %[[VAL_8]] : index +// CHECK: scf.yield %[[VAL_141]]#0, %[[VAL_141]]#1, %[[VAL_144]] : index, i1, index // CHECK: } -// CHECK: %[[VAL_167:.*]] = arith.addi %[[VAL_89]], %[[VAL_7]] : index -// CHECK: %[[VAL_168:.*]] = arith.cmpi ugt, %[[VAL_169:.*]]#2, %[[VAL_167]] : index -// CHECK: %[[VAL_170:.*]] = arith.select %[[VAL_168]], %[[VAL_169]]#2, %[[VAL_167]] : index -// CHECK: %[[VAL_171:.*]] = arith.addi %[[VAL_170]], %[[VAL_6]] : index -// CHECK: %[[VAL_172:.*]] = arith.cmpi ule, %[[VAL_171]], %[[VAL_5]] : index -// CHECK: %[[VAL_173:.*]] = arith.andi %[[VAL_169]]#1, %[[VAL_172]] : i1 -// CHECK: scf.yield %[[VAL_173]], %[[VAL_169]]#0, %[[VAL_170]], %[[VAL_135]] : i1, index, index, tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_145:.*]] = arith.addi %[[VAL_75]], %[[VAL_7]] : index +// CHECK: %[[VAL_146:.*]] = arith.cmpi ugt, %[[VAL_147:.*]]#2, %[[VAL_145]] : index +// CHECK: %[[VAL_148:.*]] = arith.select %[[VAL_146]], %[[VAL_147]]#2, %[[VAL_145]] : index +// CHECK: %[[VAL_149:.*]] = arith.addi %[[VAL_148]], %[[VAL_6]] : index +// CHECK: %[[VAL_150:.*]] = arith.cmpi ule, %[[VAL_149]], %[[VAL_5]] : index +// CHECK: %[[VAL_151:.*]] = arith.andi %[[VAL_147]]#1, %[[VAL_150]] : i1 +// CHECK: scf.yield %[[VAL_151]], %[[VAL_147]]#0, %[[VAL_148]], %[[VAL_115]] : i1, index, index, tensor<6x6xi32, #sparse> // CHECK: } -// CHECK: memref.store %[[VAL_10]], %[[VAL_20]]{{\[}}%[[VAL_7]]] : memref<5xindex> -// CHECK: %[[VAL_174:.*]] = arith.cmpi ugt, %[[VAL_33]], %[[VAL_34]] : index -// CHECK: %[[VAL_175:.*]]:3 = scf.if %[[VAL_174]] -> (index, i1, index) { -// CHECK: %[[VAL_176:.*]] = arith.addi %[[VAL_34]], %[[VAL_7]] : index -// CHECK: scf.yield %[[VAL_33]], %[[VAL_2]], %[[VAL_176]] : index, i1, index +// CHECK: %[[VAL_152:.*]] = arith.cmpi ugt, %[[VAL_31]], %[[VAL_32]] : index +// CHECK: %[[VAL_153:.*]]:3 = scf.if %[[VAL_152]] -> (index, i1, index) { +// CHECK: %[[VAL_154:.*]] = arith.addi %[[VAL_32]], %[[VAL_7]] : index +// CHECK: scf.yield %[[VAL_31]], %[[VAL_2]], %[[VAL_154]] : index, i1, index // CHECK: } else { -// CHECK: %[[VAL_177:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_10]]] : memref<5xindex> -// CHECK: %[[VAL_178:.*]]:2 = scf.for %[[VAL_179:.*]] = %[[VAL_10]] to %[[VAL_177]] step %[[VAL_7]] iter_args(%[[VAL_180:.*]] = %[[VAL_5]], %[[VAL_181:.*]] = %[[VAL_12]]) -> (index, i1) { -// CHECK: %[[VAL_182:.*]] = arith.addi %[[VAL_179]], %[[VAL_9]] : index -// CHECK: %[[VAL_183:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_182]]] : memref<5xindex> -// CHECK: %[[VAL_184:.*]] = arith.addi %[[VAL_179]], %[[VAL_6]] : index -// CHECK: %[[VAL_185:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_184]]] : memref<5xindex> -// CHECK: %[[VAL_186:.*]] = arith.cmpi ult, %[[VAL_183]], %[[VAL_185]] : index -// CHECK: %[[VAL_187:.*]] = scf.if %[[VAL_186]] -> (index) { -// CHECK: %[[VAL_188:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_183]]] : memref -// CHECK: %[[VAL_189:.*]] = arith.cmpi eq, %[[VAL_188]], %[[VAL_33]] : index -// CHECK: %[[VAL_190:.*]] = scf.if %[[VAL_189]] -> (index) { -// CHECK: %[[VAL_191:.*]] = arith.addi %[[VAL_183]], %[[VAL_7]] : index -// CHECK: memref.store %[[VAL_191]], %[[VAL_20]]{{\[}}%[[VAL_182]]] : memref<5xindex> -// CHECK: scf.yield %[[VAL_191]] : index -// CHECK: } else { -// CHECK: scf.yield %[[VAL_183]] : index -// CHECK: } -// CHECK: scf.yield %[[VAL_190]] : index -// CHECK: } else { -// CHECK: scf.yield %[[VAL_183]] : index -// CHECK: } -// CHECK: %[[VAL_192:.*]] = arith.cmpi ult, %[[VAL_187]], %[[VAL_185]] : index -// CHECK: %[[VAL_193:.*]] = scf.if %[[VAL_192]] -> (index) { -// CHECK: %[[VAL_194:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_187]]] : memref -// CHECK: scf.yield %[[VAL_194]] : index +// CHECK: %[[VAL_155:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_8]]] : memref<3xindex> +// CHECK: %[[VAL_156:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_7]]] : memref<3xindex> +// CHECK: %[[VAL_157:.*]] = arith.cmpi ult, %[[VAL_155]], %[[VAL_156]] : index +// CHECK: %[[VAL_158:.*]] = scf.if %[[VAL_157]] -> (index) { +// CHECK: %[[VAL_159:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_155]]] : memref +// CHECK: %[[VAL_160:.*]] = arith.cmpi eq, %[[VAL_159]], %[[VAL_31]] : index +// CHECK: %[[VAL_161:.*]] = scf.if %[[VAL_160]] -> (index) { +// CHECK: %[[VAL_162:.*]] = arith.addi %[[VAL_155]], %[[VAL_7]] : index +// CHECK: memref.store %[[VAL_162]], %[[VAL_18]]{{\[}}%[[VAL_8]]] : memref<3xindex> +// CHECK: scf.yield %[[VAL_162]] : index // CHECK: } else { -// CHECK: scf.yield %[[VAL_180]] : index +// CHECK: scf.yield %[[VAL_155]] : index // CHECK: } -// CHECK: %[[VAL_195:.*]] = arith.ori %[[VAL_192]], %[[VAL_181]] : i1 -// CHECK: %[[VAL_196:.*]] = arith.cmpi ult, %[[VAL_193]], %[[VAL_180]] : index -// CHECK: %[[VAL_197:.*]] = arith.select %[[VAL_196]], %[[VAL_193]], %[[VAL_180]] : index -// CHECK: scf.yield %[[VAL_197]], %[[VAL_195]] : index, i1 +// CHECK: scf.yield %[[VAL_161]] : index +// CHECK: } else { +// CHECK: scf.yield %[[VAL_155]] : index +// CHECK: } +// CHECK: %[[VAL_163:.*]] = arith.cmpi ult, %[[VAL_158]], %[[VAL_156]] : index +// CHECK: %[[VAL_164:.*]] = scf.if %[[VAL_163]] -> (index) { +// CHECK: %[[VAL_165:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_158]]] : memref +// CHECK: scf.yield %[[VAL_165]] : index +// CHECK: } else { +// CHECK: scf.yield %[[VAL_5]] : index // CHECK: } -// CHECK: %[[VAL_198:.*]] = arith.addi %[[VAL_199:.*]]#0, %[[VAL_7]] : index -// CHECK: %[[VAL_200:.*]] = arith.addi %[[VAL_199]]#0, %[[VAL_3]] : index -// CHECK: %[[VAL_201:.*]] = arith.cmpi uge, %[[VAL_198]], %[[VAL_6]] : index -// CHECK: %[[VAL_202:.*]] = arith.select %[[VAL_201]], %[[VAL_200]], %[[VAL_10]] : index -// CHECK: scf.yield %[[VAL_199]]#0, %[[VAL_199]]#1, %[[VAL_202]] : index, i1, index +// CHECK: %[[VAL_166:.*]] = arith.cmpi ult, %[[VAL_164]], %[[VAL_5]] : index +// CHECK: %[[VAL_167:.*]] = arith.select %[[VAL_166]], %[[VAL_164]], %[[VAL_5]] : index +// CHECK: %[[VAL_168:.*]] = arith.addi %[[VAL_167]], %[[VAL_7]] : index +// CHECK: %[[VAL_169:.*]] = arith.addi %[[VAL_167]], %[[VAL_3]] : index +// CHECK: %[[VAL_170:.*]] = arith.cmpi uge, %[[VAL_168]], %[[VAL_6]] : index +// CHECK: %[[VAL_171:.*]] = arith.select %[[VAL_170]], %[[VAL_169]], %[[VAL_8]] : index +// CHECK: scf.yield %[[VAL_167]], %[[VAL_163]], %[[VAL_171]] : index, i1, index // CHECK: } -// CHECK: %[[VAL_203:.*]] = arith.addi %[[VAL_34]], %[[VAL_7]] : index -// CHECK: %[[VAL_204:.*]] = arith.cmpi ugt, %[[VAL_205:.*]]#2, %[[VAL_203]] : index -// CHECK: %[[VAL_206:.*]] = arith.select %[[VAL_204]], %[[VAL_205]]#2, %[[VAL_203]] : index -// CHECK: %[[VAL_207:.*]] = arith.addi %[[VAL_206]], %[[VAL_6]] : index -// CHECK: %[[VAL_208:.*]] = arith.cmpi ule, %[[VAL_207]], %[[VAL_5]] : index -// CHECK: %[[VAL_209:.*]] = arith.andi %[[VAL_205]]#1, %[[VAL_208]] : i1 -// CHECK: scf.yield %[[VAL_209]], %[[VAL_205]]#0, %[[VAL_206]], %[[VAL_210:.*]]#2 : i1, index, index, tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_172:.*]] = arith.addi %[[VAL_32]], %[[VAL_7]] : index +// CHECK: %[[VAL_173:.*]] = arith.cmpi ugt, %[[VAL_174:.*]]#2, %[[VAL_172]] : index +// CHECK: %[[VAL_175:.*]] = arith.select %[[VAL_173]], %[[VAL_174]]#2, %[[VAL_172]] : index +// CHECK: %[[VAL_176:.*]] = arith.addi %[[VAL_175]], %[[VAL_6]] : index +// CHECK: %[[VAL_177:.*]] = arith.cmpi ule, %[[VAL_176]], %[[VAL_5]] : index +// CHECK: %[[VAL_178:.*]] = arith.andi %[[VAL_174]]#1, %[[VAL_177]] : i1 +// CHECK: scf.yield %[[VAL_178]], %[[VAL_174]]#0, %[[VAL_175]], %[[VAL_179:.*]]#2 : i1, index, index, tensor<6x6xi32, #sparse> // CHECK: } -// CHECK: %[[VAL_211:.*]] = sparse_tensor.load %[[VAL_212:.*]]#2 hasInserts : tensor<6x6xi32, #sparse{{[0-9]*}}> -// CHECK: return %[[VAL_211]] : tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_180:.*]] = sparse_tensor.load %[[VAL_181:.*]]#2 hasInserts : tensor<6x6xi32, #sparse> +// CHECK: return %[[VAL_180]] : tensor<6x6xi32, #sparse> // CHECK: } func.func @conv2d_all_sparse_CSR(%arg0: tensor<8x8xi32, #DCSR>, %arg1: tensor<3x3xi32>) -> tensor<6x6xi32, #DCSR> { diff --git a/mlir/test/Dialect/SparseTensor/spy_sddmm_bsr.mlir b/mlir/test/Dialect/SparseTensor/spy_sddmm_bsr.mlir new file mode 100755 index 0000000000000000000000000000000000000000..ed8d639878967766f20ca72e085db326d3515b8c --- /dev/null +++ b/mlir/test/Dialect/SparseTensor/spy_sddmm_bsr.mlir @@ -0,0 +1,103 @@ +// RUN: mlir-opt %s --sparse-reinterpret-map -sparsification | FileCheck %s + +// +// A SDDMM implementation with "spy" function and +// in-place update of the sampling sparse matrix. +// + +#BSR = #sparse_tensor.encoding<{ + map = (i, j) -> ( + i floordiv 2 : dense, + j floordiv 2 : compressed, + i mod 2 : dense, + j mod 2 : dense) +}> + +#trait_SDDMM = { + indexing_maps = [ + affine_map<(i,j,k) -> (i,k)>, // A + affine_map<(i,j,k) -> (k,j)>, // B + affine_map<(i,j,k) -> (i,j)> // S (in/out) + ], + iterator_types = ["parallel", "parallel", "reduction"], + doc = "S(i,j) += spy[S(i,j)] x SUM_k A(i,k) B(k,j)" +} + +// +// CHECK: #[[$BSR:.+]] = #sparse_tensor.encoding<{ map = (d0, d1) -> (d0 floordiv 2 : dense, d1 floordiv 2 : compressed, d0 mod 2 : dense, d1 mod 2 : dense) }> +// CHECK: #[[$MAP:.+]] = #sparse_tensor.encoding<{ map = (d0, d1, d2, d3) -> (d0 : dense, d1 : compressed, d2 : dense, d3 : dense) }> +// +// CHECK-LABEL: func.func @SDDMM_block( +// CHECK-SAME: %[[VAL_0:.*]]: tensor, +// CHECK-SAME: %[[VAL_1:.*]]: tensor, +// CHECK-SAME: %[[VAL_2:.*]]: tensor) -> tensor { +// CHECK-DAG: %[[VAL_3:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[VAL_5:.*]] = arith.constant 2 : index +// CHECK-DAG: %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[VAL_7:.*]] = sparse_tensor.reinterpret_map %[[VAL_0]] : tensor to tensor +// CHECK: %[[VAL_8:.*]] = tensor.dim %[[VAL_1]], %[[VAL_3]] : tensor +// CHECK: %[[VAL_9:.*]] = bufferization.to_memref %[[VAL_1]] : memref +// CHECK: %[[VAL_10:.*]] = bufferization.to_memref %[[VAL_2]] : memref +// CHECK: %[[VAL_11:.*]] = sparse_tensor.lvl %[[VAL_7]], %[[VAL_4]] : tensor +// CHECK: %[[VAL_12:.*]] = sparse_tensor.positions %[[VAL_7]] {level = 1 : index} : tensor to memref +// CHECK: %[[VAL_13:.*]] = sparse_tensor.coordinates %[[VAL_7]] {level = 1 : index} : tensor to memref +// CHECK: %[[VAL_14:.*]] = sparse_tensor.values %[[VAL_7]] : tensor to memref +// CHECK: scf.for %[[VAL_15:.*]] = %[[VAL_4]] to %[[VAL_11]] step %[[VAL_3]] { +// CHECK: %[[VAL_16:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_15]]] : memref +// CHECK: %[[VAL_17:.*]] = arith.addi %[[VAL_15]], %[[VAL_3]] : index +// CHECK: %[[VAL_18:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_17]]] : memref +// CHECK: scf.for %[[VAL_19:.*]] = %[[VAL_16]] to %[[VAL_18]] step %[[VAL_3]] { +// CHECK: %[[VAL_20:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_19]]] : memref +// CHECK: scf.for %[[VAL_21:.*]] = %[[VAL_4]] to %[[VAL_5]] step %[[VAL_3]] { +// CHECK: %[[VAL_22:.*]] = arith.muli %[[VAL_19]], %[[VAL_5]] : index +// CHECK: %[[VAL_23:.*]] = arith.addi %[[VAL_22]], %[[VAL_21]] : index +// CHECK: scf.for %[[VAL_24:.*]] = %[[VAL_4]] to %[[VAL_5]] step %[[VAL_3]] { +// CHECK: %[[VAL_25:.*]] = arith.muli %[[VAL_23]], %[[VAL_5]] : index +// CHECK: %[[VAL_26:.*]] = arith.addi %[[VAL_25]], %[[VAL_24]] : index +// CHECK: %[[VAL_27:.*]] = scf.for %[[VAL_28:.*]] = %[[VAL_4]] to %[[VAL_8]] step %[[VAL_3]] iter_args(%[[VAL_29:.*]] = %[[VAL_6]]) -> (f32) { +// CHECK: %[[VAL_30:.*]] = arith.muli %[[VAL_15]], %[[VAL_5]] : index +// CHECK: %[[VAL_31:.*]] = arith.addi %[[VAL_30]], %[[VAL_21]] : index +// CHECK: %[[VAL_32:.*]] = memref.load %[[VAL_9]]{{\[}}%[[VAL_31]], %[[VAL_28]]] : memref +// CHECK: %[[VAL_33:.*]] = arith.muli %[[VAL_20]], %[[VAL_5]] : index +// CHECK: %[[VAL_34:.*]] = arith.addi %[[VAL_33]], %[[VAL_24]] : index +// CHECK: %[[VAL_35:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_28]], %[[VAL_34]]] : memref +// CHECK: %[[VAL_36:.*]] = arith.mulf %[[VAL_32]], %[[VAL_35]] : f32 +// CHECK: %[[VAL_37:.*]] = arith.addf %[[VAL_29]], %[[VAL_36]] : f32 +// CHECK: scf.yield %[[VAL_37]] : f32 +// CHECK: } {"Emitted from" = "linalg.generic"} +// CHECK: memref.store %[[VAL_27]], %[[VAL_14]]{{\[}}%[[VAL_26]]] : memref +// CHECK: } {"Emitted from" = "linalg.generic"} +// CHECK: } {"Emitted from" = "linalg.generic"} +// CHECK: } {"Emitted from" = "linalg.generic"} +// CHECK: } {"Emitted from" = "linalg.generic"} +// CHECK: %[[VAL_38:.*]] = sparse_tensor.load %[[VAL_7]] : tensor +// CHECK: %[[VAL_39:.*]] = sparse_tensor.reinterpret_map %[[VAL_38]] : tensor to tensor +// CHECK: return %[[VAL_39]] : tensor +// CHECK: } +module { + func.func @SDDMM_block(%args: tensor, + %arga: tensor, + %argb: tensor) -> tensor { + %result = linalg.generic #trait_SDDMM + ins(%arga, %argb: tensor, tensor) + outs(%args: tensor) { + ^bb(%a: f32, %b: f32, %s: f32): + %f0 = arith.constant 0.0 : f32 + %u = sparse_tensor.unary %s : f32 to f32 + present={ + ^bb0(%p: f32): + %mul = arith.mulf %a, %b : f32 + sparse_tensor.yield %mul : f32 + } + absent={} + %r = sparse_tensor.reduce %s, %u, %f0 : f32 { + ^bb0(%p: f32, %q: f32): + %add = arith.addf %p, %q : f32 + sparse_tensor.yield %add : f32 + } + linalg.yield %r : f32 + } -> tensor + return %result : tensor + } +} diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir index 84c44a09aa3dd1c70ace72c61abe5c74253e6fa4..7d7d221c1e8e960fd74ab097b6f249504ecb5ccc 100644 --- a/mlir/test/Dialect/Tensor/canonicalize.mlir +++ b/mlir/test/Dialect/Tensor/canonicalize.mlir @@ -2018,3 +2018,20 @@ func.func @invalid_slice_ops(%t: tensor, %t2: tensor) -> tensor into tensor return %1 : tensor } + +// ----- + +// CHECK-LABEL: func @generate_negative_size_verifies( +// CHECK: %[[c:.*]] = arith.constant -8 : index +// CHECK: tensor.generate %[[c]] +// CHECK: : tensor +func.func @generate_negative_size_verifies() -> tensor { + %cst = arith.constant 0 : i32 + %c0 = arith.constant 0 : index + %size = affine.max affine_map<(d0) -> (d0 mod 64 - 8)>(%c0) + %tensor = tensor.generate %size { + ^bb0(%arg0: index, %arg1: index): + tensor.yield %cst : i32 + } : tensor + return %tensor : tensor +} diff --git a/mlir/test/Dialect/Tensor/invalid-canonicalize.mlir b/mlir/test/Dialect/Tensor/invalid-canonicalize.mlir deleted file mode 100644 index decfd55eacc9577d0fe0659daab7e682a2f86e3b..0000000000000000000000000000000000000000 --- a/mlir/test/Dialect/Tensor/invalid-canonicalize.mlir +++ /dev/null @@ -1,15 +0,0 @@ -// RUN: mlir-opt <%s -split-input-file -verify-diagnostics -canonicalize - -// ----- - -func.func @indirectly_generate_negative_size() -> tensor { - %cst = arith.constant 0 : i32 - %c0 = arith.constant 0 : index - %size = affine.max affine_map<(d0) -> (d0 mod 64 - 8)>(%c0) - // expected-error@+1 {{tensor dimensions must be non-negative}} - %tensor = tensor.generate %size { - ^bb0(%arg0: index, %arg1: index): - tensor.yield %cst : i32 - } : tensor - return %tensor : tensor -} diff --git a/mlir/test/Dialect/Tensor/invalid.mlir b/mlir/test/Dialect/Tensor/invalid.mlir index 9b6c2327879cf9b8826295e46dff54cbc503c3af..bdada43e325c5572c75ff7b507b93cd3b55c7cb8 100644 --- a/mlir/test/Dialect/Tensor/invalid.mlir +++ b/mlir/test/Dialect/Tensor/invalid.mlir @@ -163,19 +163,6 @@ func.func @tensor.generate(%m : index, %n : index) // ----- -func.func @generate_negative_size() -> tensor { - %cst = arith.constant 0 : i32 - %size = index.constant -128 - // expected-error@+1 {{tensor dimensions must be non-negative}} - %tensor = tensor.generate %size { - ^bb0(%arg0: index, %arg1: index): - tensor.yield %cst : i32 - } : tensor - return %tensor : tensor -} - -// ----- - func.func @tensor.reshape_element_type_mismatch( %buf: tensor<*xf32>, %shape: tensor<1xi32>) { // expected-error @+1 {{element types of source and destination tensor types should be the same}} diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir index edb2689364a9876bf2fe962bf53b7a10f3126759..ad248d1e14e72e09603a689cdfaf329c842e06da 100644 --- a/mlir/test/Dialect/Vector/invalid.mlir +++ b/mlir/test/Dialect/Vector/invalid.mlir @@ -332,6 +332,28 @@ func.func @test_vector.transfer_read(%arg0: memref) { // ----- +#map1 = affine_map<(d0, d1, d2) -> (d0, 0, 0)> +func.func @main(%m: memref<1xi32>, %2: vector<1x32xi1>) -> vector<1x32xi32> { + %0 = arith.constant 1 : index + %1 = arith.constant 1 : i32 + // expected-error@+1 {{expected the same rank for the vector and the results of the permutation map}} + %3 = vector.transfer_read %m[%0], %1, %2 { permutation_map = #map1 } : memref<1xi32>, vector<1x32xi32> + return %3 : vector<1x32xi32> +} + +// ----- + +#map1 = affine_map<(d0, d1, d2) -> (d0, 0, 0)> +func.func @test_vector.transfer_write(%m: memref<1xi32>, %2: vector<1x32xi32>) -> vector<1x32xi32> { + %0 = arith.constant 1 : index + %1 = arith.constant 1 : i32 + // expected-error@+1 {{expected the same rank for the vector and the results of the permutation map}} + %3 = vector.transfer_write %2, %m[%0], %1 { permutation_map = #map1 } : vector<1x32xi32>, memref<1xi32> + return %3 : vector<1x32xi32> +} + +// ----- + func.func @test_vector.transfer_read(%arg0: vector<4x3xf32>) { %c3 = arith.constant 3 : index %f0 = arith.constant 0.0 : f32 diff --git a/mlir/test/IR/custom-print-parse.mlir b/mlir/test/IR/custom-print-parse.mlir new file mode 100644 index 0000000000000000000000000000000000000000..b157fd1b1ea3386156798ebb33ffaa50ef095e0e --- /dev/null +++ b/mlir/test/IR/custom-print-parse.mlir @@ -0,0 +1,17 @@ +// RUN: mlir-opt %s | FileCheck %s + +// CHECK-LABEL: module @dimension_list +module @dimension_list { + // CHECK: test.custom_dimension_list_attr dimension_list = [] + test.custom_dimension_list_attr dimension_list = [] + // CHECK: test.custom_dimension_list_attr dimension_list = 3 + test.custom_dimension_list_attr dimension_list = 3 + // CHECK: test.custom_dimension_list_attr dimension_list = 0 + test.custom_dimension_list_attr dimension_list = 0 + // CHECK: test.custom_dimension_list_attr dimension_list = 1x2 + test.custom_dimension_list_attr dimension_list = 1x2 + // CHECK: test.custom_dimension_list_attr dimension_list = ? + test.custom_dimension_list_attr dimension_list = ? + // CHECK: test.custom_dimension_list_attr dimension_list = ?x? + test.custom_dimension_list_attr dimension_list = ?x? +} diff --git a/mlir/test/IR/invalid-custom-print-parse.mlir b/mlir/test/IR/invalid-custom-print-parse.mlir new file mode 100644 index 0000000000000000000000000000000000000000..456b16c91bc04b73587360ebe08b9c3f606c0a4e --- /dev/null +++ b/mlir/test/IR/invalid-custom-print-parse.mlir @@ -0,0 +1,16 @@ +// RUN: mlir-opt %s -split-input-file -verify-diagnostics + +// expected-error@+2 {{invalid dimension}} +// expected-error@+1 {{custom op 'test.custom_dimension_list_attr' Failed parsing dimension list.}} +test.custom_dimension_list_attr dimension_list = 1x-1 + +// ----- + +// expected-error@+1 {{custom op 'test.custom_dimension_list_attr' Failed parsing dimension list. Did you mean an empty list? It must be denoted by "[]".}} +test.custom_dimension_list_attr dimension_list = -1 + +// ----- + +// expected-error@+2 {{expected ']'}} +// expected-error@+1 {{custom op 'test.custom_dimension_list_attr' Failed parsing dimension list.}} +test.custom_dimension_list_attr dimension_list = [2x3] diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir index 7825e8fe9bafa41b4885cf9751d14cfb78bee023..6c35e2b51ed8f475d6d4210905cd23522555e009 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir @@ -41,8 +41,8 @@ module { func.func @conv2d(%input: tensor<8x8xi32>, - %filter: tensor<3x3xi32>, - %output: tensor<6x6xi32>) -> tensor<6x6xi32> { + %filter: tensor<3x3xi32>, + %output: tensor<6x6xi32>) -> tensor<6x6xi32> { %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32>, tensor<3x3xi32>) outs (%output: tensor<6x6xi32>) -> tensor<6x6xi32> @@ -50,7 +50,7 @@ module { } func.func @conv2d_all_sparse_DCSR(%input: tensor<8x8xi32, #DCSR>, - %filter: tensor<3x3xi32, #DCSR>) -> tensor<6x6xi32, #DCSR> { + %filter: tensor<3x3xi32, #DCSR>) -> tensor<6x6xi32, #DCSR> { %s = tensor.empty() : tensor<6x6xi32, #DCSR> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #DCSR>, tensor<3x3xi32, #DCSR>) @@ -59,7 +59,7 @@ module { } func.func @conv2d_all_sparse_CSR(%input: tensor<8x8xi32, #CSR>, - %filter: tensor<3x3xi32, #CSR>) -> tensor<6x6xi32, #CSR> { + %filter: tensor<3x3xi32, #CSR>) -> tensor<6x6xi32, #CSR> { %s = tensor.empty() : tensor<6x6xi32, #CSR> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #CSR>, tensor<3x3xi32, #CSR>) @@ -68,7 +68,7 @@ module { } func.func @conv2d_all_sparse_CD(%input: tensor<8x8xi32, #CDR>, - %filter: tensor<3x3xi32, #CDR>) -> tensor<6x6xi32, #CDR> { + %filter: tensor<3x3xi32, #CDR>) -> tensor<6x6xi32, #CDR> { %s = tensor.empty() : tensor<6x6xi32, #CDR> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #CDR>, tensor<3x3xi32, #CDR>) @@ -77,7 +77,7 @@ module { } func.func @conv2d_all_sparse_CSC(%input: tensor<8x8xi32, #CSC>, - %filter: tensor<3x3xi32, #CSC>) -> tensor<6x6xi32, #CSC> { + %filter: tensor<3x3xi32, #CSC>) -> tensor<6x6xi32, #CSC> { %s = tensor.empty() : tensor<6x6xi32, #CSC> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #CSC>, tensor<3x3xi32, #CSC>) diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir index c9abaa2aff63db44a6062b7bca270e8030d9dc69..9295ee8b469c2fcd58fc9b6a8556aa7f62922453 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir @@ -25,7 +25,6 @@ // // Do the same run, but now with direct IR generation and vectorization. // REDEFINE: %{sparsifier_opts} = enable-runtime-library=false enable-buffer-initialization=true vl=2 reassociate-fp-reductions=true enable-index-optimizations=true - // RUN: %{compile} | %{run} | FileCheck %s // // Do the same run, but now with direct IR generation and VLA vectorization. diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir index 80946f5388520a89167becd1d1a3ede6e76885ff..41071ea700fb604fff6eee714e82bb427d1cff91 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir @@ -25,7 +25,6 @@ // // Do the same run, but now with direct IR generation and vectorization. // REDEFINE: %{sparsifier_opts} = enable-runtime-library=false vl=2 reassociate-fp-reductions=true enable-index-optimizations=true - // RUN: %{compile} | %{run} | FileCheck %s // // Do the same run, but now with direct IR generation and VLA vectorization. @@ -46,8 +45,8 @@ module { func.func @conv2d(%input: tensor<8x8xi32>, - %filter: tensor<3x3xi32>, - %output: tensor<6x6xi32>) -> tensor<6x6xi32> { + %filter: tensor<3x3xi32>, + %output: tensor<6x6xi32>) -> tensor<6x6xi32> { %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32>, tensor<3x3xi32>) outs (%output: tensor<6x6xi32>) -> tensor<6x6xi32> @@ -56,21 +55,21 @@ module { func.func @conv2d_CSR_dense_rotated(%arg0: tensor<8x8xi32, #CSR>, %arg1: tensor<3x3xi32>) -> tensor<6x6xi32> { - %s = tensor.empty() : tensor<6x6xi32> + %s = arith.constant dense<0> : tensor<6x6xi32> %0 = linalg.generic {indexing_maps = [#map, #map1, #map2], - iterator_types = ["parallel", "reduction", "reduction", "parallel"]} - ins(%arg0, %arg1 : tensor<8x8xi32, #CSR>, tensor<3x3xi32>) - outs(%s : tensor<6x6xi32>) attrs = {sorted = true} { - ^bb0(%in: i32, %in_0: i32, %out: i32): - %1 = arith.muli %in, %in_0 : i32 - %2 = arith.addi %out, %1 : i32 - linalg.yield %2 : i32 + iterator_types = ["parallel", "reduction", "reduction", "parallel"]} + ins(%arg0, %arg1 : tensor<8x8xi32, #CSR>, tensor<3x3xi32>) + outs(%s : tensor<6x6xi32>) attrs = {sorted = true} { + ^bb0(%in: i32, %in_0: i32, %out: i32): + %1 = arith.muli %in, %in_0 : i32 + %2 = arith.addi %out, %1 : i32 + linalg.yield %2 : i32 } -> tensor<6x6xi32> return %0 : tensor<6x6xi32> } func.func @conv2d_sparse_out(%input: tensor<8x8xi32>, - %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #DCSR> { + %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #DCSR> { %s = tensor.empty() : tensor<6x6xi32, #DCSR> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32>, tensor<3x3xi32>) @@ -79,7 +78,7 @@ module { } func.func @conv2d_all_sparse_DCSR(%input: tensor<8x8xi32, #DCSR>, - %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #DCSR> { + %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #DCSR> { %s = tensor.empty() : tensor<6x6xi32, #DCSR> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #DCSR>, tensor<3x3xi32>) @@ -88,7 +87,7 @@ module { } func.func @conv2d_all_sparse_CSR(%input: tensor<8x8xi32, #CSR>, - %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #CSR> { + %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #CSR> { %s = tensor.empty() : tensor<6x6xi32, #CSR> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #CSR>, tensor<3x3xi32>) @@ -97,7 +96,7 @@ module { } func.func @conv2d_all_sparse_CD(%input: tensor<8x8xi32, #CDR>, - %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #CDR> { + %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #CDR> { %s = tensor.empty() : tensor<6x6xi32, #CDR> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #CDR>, tensor<3x3xi32>) @@ -106,7 +105,7 @@ module { } func.func @conv2d_all_sparse_CSC(%input: tensor<8x8xi32, #CSC>, - %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #CSC> { + %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #CSC> { %s = tensor.empty() : tensor<6x6xi32, #CSC> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #CSC>, tensor<3x3xi32>) @@ -125,7 +124,6 @@ module { [ -1, 0, 1 ] ]> : tensor<3x3xi32> - %input = arith.constant dense<[ [ 1, 2, 3, 4, 0, 6, 7, 8 ], [ 2, 2, 4, 4, 0, 0, 6, 8 ], @@ -270,7 +268,6 @@ module { : tensor<6x6xi32>, vector<6x6xi32> vector.print %v : vector<6x6xi32> - // Release the resources. bufferization.dealloc_tensor %sparse_input_DCSR : tensor<8x8xi32, #DCSR> bufferization.dealloc_tensor %sparse_input_CSR : tensor<8x8xi32, #CSR> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir new file mode 100755 index 0000000000000000000000000000000000000000..9e7bbe71038b114eb0ca808da03afd8126d0fcb9 --- /dev/null +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir @@ -0,0 +1,220 @@ +//-------------------------------------------------------------------------------------------------- +// WHEN CREATING A NEW TEST, PLEASE JUST COPY & PASTE WITHOUT EDITS. +// +// Set-up that's shared across all tests in this directory. In principle, this +// config could be moved to lit.local.cfg. However, there are downstream users that +// do not use these LIT config files. Hence why this is kept inline. +// +// DEFINE: %{sparsifier_opts} = enable-runtime-library=true +// DEFINE: %{sparsifier_opts_sve} = enable-arm-sve=true %{sparsifier_opts} +// DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" +// DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" +// DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils +// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} +// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} +// +// DEFINE: %{env} = +//-------------------------------------------------------------------------------------------------- + +// RUN: %{compile} | %{run} | FileCheck %s +// +// Do the same run, but now with direct IR generation. +// REDEFINE: %{sparsifier_opts} = enable-runtime-library=false +// RUN: %{compile} | %{run} | FileCheck %s +// +// Do the same run, but now with direct IR generation and vectorization. +// REDEFINE: %{sparsifier_opts} = enable-runtime-library=false vl=2 reassociate-fp-reductions=true enable-index-optimizations=true +// +// Do the same run, but now with direct IR generation and VLA vectorization. +// RUN: %if mlir_arm_sve_tests %{ %{compile_sve} | %{run_sve} | FileCheck %s %} + +#CSR = #sparse_tensor.encoding<{ + map = (d0, d1) -> (d0 : dense, + d1 : compressed) +}> + +#DCSR = #sparse_tensor.encoding<{ + map = (d0, d1) -> (d0 : compressed, + d1 : compressed) +}> + +// An example of a 2D convolution with sparse data and filter. +module { + func.func @conv2d(%input: tensor<10x10xi32>, + %filter: tensor<5x5xi32>, + %output: tensor<6x6xi32>) -> tensor<6x6xi32> { + %0 = linalg.conv_2d + ins (%input, %filter: tensor<10x10xi32>, tensor<5x5xi32>) + outs (%output: tensor<6x6xi32>) -> tensor<6x6xi32> + return %0 : tensor<6x6xi32> + } + + func.func @conv2d_ss(%input: tensor<10x10xi32, #CSR>, + %filter: tensor<5x5xi32, #CSR>, + %output: tensor<6x6xi32>) -> tensor<6x6xi32> { + %0 = linalg.conv_2d + ins (%input, %filter: tensor<10x10xi32, #CSR>, tensor<5x5xi32, #CSR>) + outs (%output: tensor<6x6xi32>) -> tensor<6x6xi32> + return %0 : tensor<6x6xi32> + } + + func.func @conv2d_bs(%input: tensor<10x10xi32, #DCSR>, + %filter: tensor<5x5xi32, #CSR>, + %output: tensor<6x6xi32>) -> tensor<6x6xi32> { + %0 = linalg.conv_2d + ins (%input, %filter: tensor<10x10xi32, #DCSR>, tensor<5x5xi32, #CSR>) + outs (%output: tensor<6x6xi32>) -> tensor<6x6xi32> + return %0 : tensor<6x6xi32> + } + + func.func @entry() { + %c0 = arith.constant 0 : index + %i0 = arith.constant 0 : i32 + + // Dense filter and input to "stress" test sparsity. + + %filter = arith.constant dense<[ + [ -1, -2, -3, -4, -5 ], + [ -6, -7, -8, -9, -10 ], + [ -11, -12, -13, -14, -15 ], + [ -16, -17, -18, -19, -20 ], + [ -21, -22, -23, -24, -25 ] + ]> : tensor<5x5xi32> + + %input = arith.constant dense<[ + [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ], + [ 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ], + [ 20, 21, 22, 23, 24, 25, 26, 27, 28, 29 ], + [ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39 ], + [ 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 ], + [ 50, 51, 52, 53, 54, 55, 56, 57, 58, 59 ], + [ 60, 61, 62, 63, 64, 65, 66, 67, 68, 69 ], + [ 70, 71, 72, 73, 74, 75, 76, 77, 78, 79 ], + [ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89 ], + [ 90, 91, 92, 93, 94, 95, 96, 97, 98, 99 ] + ]> : tensor<10x10xi32> + + // Sparse filter and input to test true sparsity. + + %sfilter = arith.constant dense<[ + [ 0, -1, 0, -2, 0 ], + [ 0, 0, 0, 0, 0 ], + [ 0, 0, 8, 0, 0 ], + [ -3, 0, 0, -4, 0 ], + [ 0, 0, -5, 0, -6 ] + ]> : tensor<5x5xi32> + + %sinput = arith.constant dense<[ + [ 0, 1, 2, 3, 0, 0, 0, 0, 0, 0 ], + [ 0, 4, 0, 0, 5, 0, 0, 0, 0, 0 ], + [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], + [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], + [ 0, 0, 0, 0, 0, 0, 6, 0, 0, 7 ], + [ 0, 0, 0, 0, 0, 0, 0, 8, 0, 0 ], + [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], + [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], + [ 0, 9, 0, 0, 0, 0, 0, 0, 0, 0 ], + [ 0, 0, 0, 0, 10, 0, 0, 0, 0, 0 ] + ]> : tensor<10x10xi32> + + // Set up sparse tensors. + + %input_CSR = sparse_tensor.convert %input : tensor<10x10xi32> to tensor<10x10xi32, #CSR> + %input_DCSR = sparse_tensor.convert %input : tensor<10x10xi32> to tensor<10x10xi32, #DCSR> + %filter_CSR = sparse_tensor.convert %filter : tensor<5x5xi32> to tensor<5x5xi32, #CSR> + + %sinput_CSR = sparse_tensor.convert %sinput : tensor<10x10xi32> to tensor<10x10xi32, #CSR> + %sinput_DCSR = sparse_tensor.convert %sinput : tensor<10x10xi32> to tensor<10x10xi32, #DCSR> + %sfilter_CSR = sparse_tensor.convert %sfilter : tensor<5x5xi32> to tensor<5x5xi32, #CSR> + + // Call the kernels with stress input. + %output0 = arith.constant dense<0> : tensor<6x6xi32> + %0 = call @conv2d(%input, %filter, %output0) + : (tensor<10x10xi32>, tensor<5x5xi32>, tensor<6x6xi32>) -> tensor<6x6xi32> + %output1 = arith.constant dense<0> : tensor<6x6xi32> + %1 = call @conv2d_ss(%input_CSR, %filter_CSR, %output1) + : (tensor<10x10xi32, #CSR>, tensor<5x5xi32, #CSR>, tensor<6x6xi32>) -> tensor<6x6xi32> + %output2 = arith.constant dense<0> : tensor<6x6xi32> + %2 = call @conv2d_bs(%input_DCSR, %filter_CSR, %output2) + : (tensor<10x10xi32, #DCSR>, tensor<5x5xi32, #CSR>, tensor<6x6xi32>) -> tensor<6x6xi32> + + // Call the kernels with sparse input. + %output3 = arith.constant dense<0> : tensor<6x6xi32> + %3 = call @conv2d(%sinput, %sfilter, %output3) + : (tensor<10x10xi32>, tensor<5x5xi32>, tensor<6x6xi32>) -> tensor<6x6xi32> + %output4 = arith.constant dense<0> : tensor<6x6xi32> + %4 = call @conv2d_ss(%sinput_CSR, %sfilter_CSR, %output4) + : (tensor<10x10xi32, #CSR>, tensor<5x5xi32, #CSR>, tensor<6x6xi32>) -> tensor<6x6xi32> + %output5 = arith.constant dense<0> : tensor<6x6xi32> + %5 = call @conv2d_bs(%sinput_DCSR, %sfilter_CSR, %output5) + : (tensor<10x10xi32, #DCSR>, tensor<5x5xi32, #CSR>, tensor<6x6xi32>) -> tensor<6x6xi32> + + // Verify the output. + // + // CHECK: ( ( -9700, -10025, -10350, -10675, -11000, -11325 ), + // CHECK-SAME: ( -12950, -13275, -13600, -13925, -14250, -14575 ), + // CHECK-SAME: ( -16200, -16525, -16850, -17175, -17500, -17825 ), + // CHECK-SAME: ( -19450, -19775, -20100, -20425, -20750, -21075 ), + // CHECK-SAME: ( -22700, -23025, -23350, -23675, -24000, -24325 ), + // CHECK-SAME: ( -25950, -26275, -26600, -26925, -27250, -27575 ) ) + // + // CHECK: ( ( -9700, -10025, -10350, -10675, -11000, -11325 ), + // CHECK-SAME: ( -12950, -13275, -13600, -13925, -14250, -14575 ), + // CHECK-SAME: ( -16200, -16525, -16850, -17175, -17500, -17825 ), + // CHECK-SAME: ( -19450, -19775, -20100, -20425, -20750, -21075 ), + // CHECK-SAME: ( -22700, -23025, -23350, -23675, -24000, -24325 ), + // CHECK-SAME: ( -25950, -26275, -26600, -26925, -27250, -27575 ) ) + // + // CHECK: ( ( -9700, -10025, -10350, -10675, -11000, -11325 ), + // CHECK-SAME: ( -12950, -13275, -13600, -13925, -14250, -14575 ), + // CHECK-SAME: ( -16200, -16525, -16850, -17175, -17500, -17825 ), + // CHECK-SAME: ( -19450, -19775, -20100, -20425, -20750, -21075 ), + // CHECK-SAME: ( -22700, -23025, -23350, -23675, -24000, -24325 ), + // CHECK-SAME: ( -25950, -26275, -26600, -26925, -27250, -27575 ) ) + // + // CHECK: ( ( -7, -2, -39, 0, -30, -42 ), + // CHECK-SAME: ( -4, -10, 0, -77, 0, -40 ), + // CHECK-SAME: ( 0, 0, 0, 0, 16, 0 ), + // CHECK-SAME: ( 0, 0, 0, 0, 0, 64 ), + // CHECK-SAME: ( 0, 0, 0, -12, 0, -6 ), + // CHECK-SAME: ( -60, -27, -50, 0, -16, 0 ) ) + // + // CHECK: ( ( -7, -2, -39, 0, -30, -42 ), + // CHECK-SAME: ( -4, -10, 0, -77, 0, -40 ), + // CHECK-SAME: ( 0, 0, 0, 0, 16, 0 ), + // CHECK-SAME: ( 0, 0, 0, 0, 0, 64 ), + // CHECK-SAME: ( 0, 0, 0, -12, 0, -6 ), + // CHECK-SAME: ( -60, -27, -50, 0, -16, 0 ) ) + // + // CHECK: ( ( -7, -2, -39, 0, -30, -42 ), + // CHECK-SAME: ( -4, -10, 0, -77, 0, -40 ), + // CHECK-SAME: ( 0, 0, 0, 0, 16, 0 ), + // CHECK-SAME: ( 0, 0, 0, 0, 0, 64 ), + // CHECK-SAME: ( 0, 0, 0, -12, 0, -6 ), + // CHECK-SAME: ( -60, -27, -50, 0, -16, 0 ) ) + // + %v0 = vector.transfer_read %0[%c0, %c0], %i0 : tensor<6x6xi32>, vector<6x6xi32> + vector.print %v0 : vector<6x6xi32> + %v1 = vector.transfer_read %1[%c0, %c0], %i0 : tensor<6x6xi32>, vector<6x6xi32> + vector.print %v1 : vector<6x6xi32> + %v2 = vector.transfer_read %2[%c0, %c0], %i0 : tensor<6x6xi32>, vector<6x6xi32> + vector.print %v2 : vector<6x6xi32> + %v3 = vector.transfer_read %3[%c0, %c0], %i0 : tensor<6x6xi32>, vector<6x6xi32> + vector.print %v3 : vector<6x6xi32> + %v4 = vector.transfer_read %4[%c0, %c0], %i0 : tensor<6x6xi32>, vector<6x6xi32> + vector.print %v4 : vector<6x6xi32> + %v5 = vector.transfer_read %5[%c0, %c0], %i0 : tensor<6x6xi32>, vector<6x6xi32> + vector.print %v5 : vector<6x6xi32> + + // Release sparse resources. + bufferization.dealloc_tensor %input_CSR : tensor<10x10xi32, #CSR> + bufferization.dealloc_tensor %input_DCSR : tensor<10x10xi32, #DCSR> + bufferization.dealloc_tensor %filter_CSR : tensor<5x5xi32, #CSR> + bufferization.dealloc_tensor %sinput_CSR : tensor<10x10xi32, #CSR> + bufferization.dealloc_tensor %sinput_DCSR : tensor<10x10xi32, #DCSR> + bufferization.dealloc_tensor %sfilter_CSR : tensor<5x5xi32, #CSR> + + return + } +} diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir index e2138207f47e7fcd8f4d8098ba54cd136e7d3804..c7e4ffab7ab3b1b0c63e6c05ed1f45916684f131 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir @@ -25,7 +25,6 @@ // // Do the same run, but now with direct IR generation and vectorization. // REDEFINE: %{sparsifier_opts} = enable-runtime-library=false enable-buffer-initialization=true vl=2 reassociate-fp-reductions=true enable-index-optimizations=true - // RUN: %{compile} | %{run} | FileCheck %s // // Do the same run, but now with direct IR generation and VLA vectorization. diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir index dfb1bb71a68c41aa254d036376e812f0e6420b58..451d2b8769461487133d154b10e47ddd3f289bf4 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir @@ -38,10 +38,14 @@ map = (d0, d1, d2) -> (d0 : compressed, d1 : dense, d2 : compressed) }> -#DDC = #sparse_tensor.encoding<{ +#DCC = #sparse_tensor.encoding<{ map = (d0, d1, d2) -> (d0 : dense, d1 : compressed, d2 : compressed) }> +#DDC = #sparse_tensor.encoding<{ + map = (d0, d1, d2) -> (d0 : dense, d1 : dense, d2 : compressed) +}> + // Creates and returns 3-D buffer of size (%s1, %s2, %s3) filled with the value %f func.func @alloc_3d_filled_f32(%s1 : index, %s2 : index, %s3 : index, %f : f32) -> tensor { %buf = tensor.empty(%s1, %s2, %s3) : tensor @@ -74,6 +78,15 @@ func.func @conv_3d_CDC(%arg0: tensor, %arg1: tensor) return %ret : tensor } +func.func @conv_3d_DCC(%arg0: tensor, %arg1: tensor) -> tensor { + %c6 = arith.constant 6 : index + %s = tensor.empty(%c6, %c6, %c6) : tensor + %ret = linalg.conv_3d + ins (%arg0, %arg1: tensor, tensor) + outs (%s: tensor) -> tensor + return %ret : tensor +} + func.func @conv_3d_DDC(%arg0: tensor, %arg1: tensor) -> tensor { %c6 = arith.constant 6 : index %s = tensor.empty(%c6, %c6, %c6) : tensor @@ -102,12 +115,15 @@ func.func @entry() { : tensor to tensor %in3D_CDC = sparse_tensor.convert %in3D : tensor to tensor + %in3D_DCC = sparse_tensor.convert %in3D + : tensor to tensor %in3D_DDC = sparse_tensor.convert %in3D : tensor to tensor %dense_ret = call @conv_3d(%in3D, %filter3D, %out3D) : (tensor, tensor, tensor) -> (tensor) %CCC_ret = call @conv_3d_CCC(%in3D_CCC, %filter3D) : (tensor, tensor) -> (tensor) %CDC_ret = call @conv_3d_CDC(%in3D_CDC, %filter3D) : (tensor, tensor) -> (tensor) + %DCC_ret = call @conv_3d_DCC(%in3D_DCC, %filter3D) : (tensor, tensor) -> (tensor) %DDC_ret = call @conv_3d_DDC(%in3D_DDC, %filter3D) : (tensor, tensor) -> (tensor) // CHECK:( ( ( 108, 108, 108, 108, 108, 108 ), @@ -276,6 +292,48 @@ func.func @entry() { : tensor, vector<6x6x6xf32> vector.print %v2 : vector<6x6x6xf32> + // CHECK-NEXT:( ( ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 124, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 124, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 124, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ) ), + // CHECK-SAME: ( ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ) ), + // CHECK-SAME: ( ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ) ), + // CHECK-SAME: ( ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ) ), + // CHECK-SAME: ( ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ) ), + // CHECK-SAME: ( ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ) ) ) + %4 = sparse_tensor.convert %DCC_ret + : tensor to tensor + %v4 = vector.transfer_read %3[%c0, %c0, %c0], %zero + : tensor, vector<6x6x6xf32> + vector.print %v2 : vector<6x6x6xf32> + // Free the resources bufferization.dealloc_tensor %in3D : tensor bufferization.dealloc_tensor %filter3D : tensor @@ -284,9 +342,11 @@ func.func @entry() { bufferization.dealloc_tensor %in3D_CDC : tensor bufferization.dealloc_tensor %in3D_CCC : tensor bufferization.dealloc_tensor %in3D_DDC : tensor + bufferization.dealloc_tensor %in3D_DCC : tensor bufferization.dealloc_tensor %CCC_ret : tensor bufferization.dealloc_tensor %CDC_ret : tensor bufferization.dealloc_tensor %DDC_ret : tensor + bufferization.dealloc_tensor %DCC_ret : tensor return } diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir index e96ee4d063e77ca21e2f1b80f7726e5c78be50b7..2c4b96804ccc70badf74caf9bef1c4e79ee49632 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir @@ -25,7 +25,6 @@ // // Do the same run, but now with direct IR generation and vectorization. // REDEFINE: %{sparsifier_opts} = enable-runtime-library=false enable-buffer-initialization=true vl=2 reassociate-fp-reductions=true enable-index-optimizations=true - // RUN: %{compile} | %{run} | FileCheck %s // // Do the same run, but now with direct IR generation and VLA vectorization. diff --git a/mlir/test/Target/LLVMIR/Import/nsw_nuw.ll b/mlir/test/Target/LLVMIR/Import/nsw_nuw.ll new file mode 100644 index 0000000000000000000000000000000000000000..d08098a5e5dfe0777abe68ec7a99bb4d62ec3818 --- /dev/null +++ b/mlir/test/Target/LLVMIR/Import/nsw_nuw.ll @@ -0,0 +1,14 @@ +; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s + +; CHECK-LABEL: @intflag_inst +define void @intflag_inst(i64 %arg1, i64 %arg2) { + ; CHECK: llvm.add %{{.*}}, %{{.*}} overflow : i64 + %1 = add nsw i64 %arg1, %arg2 + ; CHECK: llvm.sub %{{.*}}, %{{.*}} overflow : i64 + %2 = sub nuw i64 %arg1, %arg2 + ; CHECK: llvm.mul %{{.*}}, %{{.*}} overflow : i64 + %3 = mul nsw nuw i64 %arg1, %arg2 + ; CHECK: llvm.shl %{{.*}}, %{{.*}} overflow : i64 + %4 = shl nuw nsw i64 %arg1, %arg2 + ret void +} diff --git a/mlir/test/Target/LLVMIR/nsw_nuw.mlir b/mlir/test/Target/LLVMIR/nsw_nuw.mlir new file mode 100644 index 0000000000000000000000000000000000000000..6843c2ef0299c7372e79f36b5765ef97b506650b --- /dev/null +++ b/mlir/test/Target/LLVMIR/nsw_nuw.mlir @@ -0,0 +1,14 @@ +// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s + +// CHECK-LABEL: define void @intflags_func +llvm.func @intflags_func(%arg0: i64, %arg1: i64) { + // CHECK: %{{.*}} = add nsw i64 %{{.*}}, %{{.*}} + %0 = llvm.add %arg0, %arg1 overflow : i64 + // CHECK: %{{.*}} = sub nuw i64 %{{.*}}, %{{.*}} + %1 = llvm.sub %arg0, %arg1 overflow : i64 + // CHECK: %{{.*}} = mul nuw nsw i64 %{{.*}}, %{{.*}} + %2 = llvm.mul %arg0, %arg1 overflow : i64 + // CHECK: %{{.*}} = shl nuw nsw i64 %{{.*}}, %{{.*}} + %3 = llvm.shl %arg0, %arg1 overflow : i64 + llvm.return +} diff --git a/mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp b/mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp index 565d07669792f1ba65b20e7666489bbdbff8fff1..a8a808424b690f60b91680a98fd241de4e30cdef 100644 --- a/mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp +++ b/mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp @@ -217,6 +217,7 @@ struct TestSCFPipeliningPass if (annotatePipeline) options.annotateFn = annotate; if (noEpiloguePeeling) { + options.supportDynamicLoops = true; options.peelEpilogue = false; options.predicateFn = predicateOp; } diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td index 1add9bd3c329438bb8bb1b3bc55d35b42f74752d..96f66c2ca06ecf6580e690c5e3c71a934c5b778f 100644 --- a/mlir/test/lib/Dialect/Test/TestOps.td +++ b/mlir/test/lib/Dialect/Test/TestOps.td @@ -2020,6 +2020,20 @@ def AffineScopeOp : TEST_Op<"affine_scope", [AffineScope]> { let hasCustomAssemblyFormat = 1; } +//===----------------------------------------------------------------------===// +// Custom printer/parser + +def CustomDimensionListAttrOp : TEST_Op<"custom_dimension_list_attr"> { + let description = [{ + Test printing/parsing of dimension list attribute. + }]; + let arguments = (ins DenseI64ArrayAttr:$dimension_list); + let assemblyFormat = [{ + `dimension_list` `=` custom($dimension_list) + attr-dict + }]; +} + //===----------------------------------------------------------------------===// // Test OpAsmInterface. diff --git a/mlir/test/lib/Transforms/TestIntRangeInference.cpp b/mlir/test/lib/Transforms/TestIntRangeInference.cpp index d1978b6099f0459f3b40c6f33e7b093ba76cbb81..2f6dd5b8095dfadeed4b90ad88e83945a6220f10 100644 --- a/mlir/test/lib/Transforms/TestIntRangeInference.cpp +++ b/mlir/test/lib/Transforms/TestIntRangeInference.cpp @@ -9,6 +9,7 @@ // functionality has been integrated into SCCP. //===----------------------------------------------------------------------===// +#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h" #include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h" #include "mlir/Analysis/DataFlow/IntegerRangeAnalysis.h" #include "mlir/Interfaces/SideEffectInterfaces.h" @@ -107,6 +108,7 @@ struct TestIntRangeInference Operation *op = getOperation(); DataFlowSolver solver; solver.load(); + solver.load(); solver.load(); if (failed(solver.initializeAndRun(op))) return signalPassFailure(); diff --git a/mlir/test/python/dialects/affine.py b/mlir/test/python/dialects/affine.py index df42f8fcf1a57d3d57d3e55b7a622ea8c2d79c35..6f39e1348fcd57c175fffb866846e964e270314d 100644 --- a/mlir/test/python/dialects/affine.py +++ b/mlir/test/python/dialects/affine.py @@ -5,6 +5,7 @@ from mlir.dialects import func from mlir.dialects import arith from mlir.dialects import memref from mlir.dialects import affine +import mlir.extras.types as T def constructAndPrintInModule(f): @@ -43,6 +44,17 @@ def testAffineStoreOp(): return mem +# CHECK-LABEL: TEST: testAffineDelinearizeInfer +@constructAndPrintInModule +def testAffineDelinearizeInfer(): + # CHECK: %[[C0:.*]] = arith.constant 0 : index + c0 = arith.ConstantOp(T.index(), 0) + # CHECK: %[[C1:.*]] = arith.constant 1 : index + c1 = arith.ConstantOp(T.index(), 1) + # CHECK: %{{.*}}:2 = affine.delinearize_index %[[C1:.*]] into (%[[C1:.*]], %[[C0:.*]]) : index, index + two_indices = affine.AffineDelinearizeIndexOp(c1, [c1, c0]) + + # CHECK-LABEL: TEST: testAffineLoadOp @constructAndPrintInModule def testAffineLoadOp(): @@ -107,66 +119,149 @@ def testAffineForOp(): # CHECK: %[[TMP:.*]] = memref.load %[[BUFFER]][%[[INDVAR]]] : memref<1024xf32> tmp = memref.LoadOp(buffer, [sum.induction_variable]) sum_next = arith.AddFOp(sum.inner_iter_args[0], tmp) - affine.AffineYieldOp([sum_next]) - return + +# CHECK-LABEL: TEST: testAffineForOpErrors +@constructAndPrintInModule +def testAffineForOpErrors(): + c1 = arith.ConstantOp(T.index(), 1) + c2 = arith.ConstantOp(T.index(), 2) + c3 = arith.ConstantOp(T.index(), 3) + d0 = AffineDimExpr.get(0) + + try: + affine.AffineForOp( + c1, + c2, + 1, + lower_bound_operands=[c3], + upper_bound_operands=[], + ) + except ValueError as e: + assert ( + e.args[0] + == "Either a concrete lower bound or an AffineMap in combination with lower bound operands, but not both, is supported." + ) + + try: + affine.AffineForOp( + AffineMap.get_constant(1), + c2, + 1, + lower_bound_operands=[c3, c3], + upper_bound_operands=[], + ) + except ValueError as e: + assert ( + e.args[0] + == "Wrong number of lower bound operands passed to AffineForOp; Expected 0, got 2." + ) + + try: + two_indices = affine.AffineDelinearizeIndexOp(c1, [c1, c1]) + affine.AffineForOp( + two_indices, + c2, + 1, + lower_bound_operands=[], + upper_bound_operands=[], + ) + except ValueError as e: + assert e.args[0] == "Only a single concrete value is supported for lower bound." + + try: + affine.AffineForOp( + 1.0, + c2, + 1, + lower_bound_operands=[], + upper_bound_operands=[], + ) + except ValueError as e: + assert e.args[0] == "lower bound must be int | ResultValueT | AffineMap." @constructAndPrintInModule def testForSugar(): - index_type = IndexType.get() - memref_t = MemRefType.get([10], index_type) + memref_t = T.memref(10, T.index()) range = affine.for_ - # CHECK: func.func @range_loop_1(%[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: index, %[[VAL_3:.*]]: memref<10xindex>) { - # CHECK: %[[VAL_4:.*]] = arith.constant 10 : index - # CHECK: affine.for %[[VAL_6:.*]] = %[[VAL_0]] to %[[VAL_4]] step 2 { - # CHECK: %[[VAL_7:.*]] = arith.addi %[[VAL_6]], %[[VAL_6]] : index - # CHECK: affine.store %[[VAL_7]], %[[VAL_3]]{{\[symbol\(}}%[[VAL_6]]{{\)\]}} : memref<10xindex> - # CHECK: } - # CHECK: return - # CHECK: } - @func.FuncOp.from_py_func(index_type, index_type, index_type, memref_t) - def range_loop_1(lb, ub, step, memref_v): - for i in range(lb, 10, 2): + # CHECK: #[[$ATTR_2:.+]] = affine_map<(d0) -> (d0)> + + # CHECK-LABEL: func.func @range_loop_1( + # CHECK-SAME: %[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: memref<10xindex>) { + # CHECK: affine.for %[[VAL_3:.*]] = #[[$ATTR_2]](%[[VAL_0]]) to #[[$ATTR_2]](%[[VAL_1]]) { + # CHECK: %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_3]] : index + # CHECK: memref.store %[[VAL_4]], %[[VAL_2]]{{\[}}%[[VAL_3]]] : memref<10xindex> + # CHECK: } + # CHECK: return + # CHECK: } + @func.FuncOp.from_py_func(T.index(), T.index(), memref_t) + def range_loop_1(lb, ub, memref_v): + for i in range(lb, ub, step=1): + add = arith.addi(i, i) + memref.store(add, memref_v, [i]) + + affine.yield_([]) + + # CHECK-LABEL: func.func @range_loop_2( + # CHECK-SAME: %[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: memref<10xindex>) { + # CHECK: affine.for %[[VAL_3:.*]] = #[[$ATTR_2]](%[[VAL_0]]) to 10 { + # CHECK: %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_3]] : index + # CHECK: memref.store %[[VAL_4]], %[[VAL_2]]{{\[}}%[[VAL_3]]] : memref<10xindex> + # CHECK: } + # CHECK: return + # CHECK: } + @func.FuncOp.from_py_func(T.index(), T.index(), memref_t) + def range_loop_2(lb, ub, memref_v): + for i in range(lb, 10, step=1): + add = arith.addi(i, i) + memref.store(add, memref_v, [i]) + affine.yield_([]) + + # CHECK-LABEL: func.func @range_loop_3( + # CHECK-SAME: %[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: memref<10xindex>) { + # CHECK: affine.for %[[VAL_3:.*]] = 0 to #[[$ATTR_2]](%[[VAL_1]]) { + # CHECK: %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_3]] : index + # CHECK: memref.store %[[VAL_4]], %[[VAL_2]]{{\[}}%[[VAL_3]]] : memref<10xindex> + # CHECK: } + # CHECK: return + # CHECK: } + @func.FuncOp.from_py_func(T.index(), T.index(), memref_t) + def range_loop_3(lb, ub, memref_v): + for i in range(0, ub, step=1): add = arith.addi(i, i) - s0 = AffineSymbolExpr.get(0) - map = AffineMap.get(0, 1, [s0]) - affine.store(add, memref_v, [i], map=map) - affine.AffineYieldOp([]) - - # CHECK: func.func @range_loop_2(%[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: index, %[[VAL_3:.*]]: memref<10xindex>) { - # CHECK: %[[VAL_4:.*]] = arith.constant 0 : index - # CHECK: %[[VAL_5:.*]] = arith.constant 10 : index - # CHECK: affine.for %[[VAL_7:.*]] = %[[VAL_4]] to %[[VAL_5]] { - # CHECK: %[[VAL_8:.*]] = arith.addi %[[VAL_7]], %[[VAL_7]] : index - # CHECK: affine.store %[[VAL_8]], %[[VAL_3]]{{\[symbol\(}}%[[VAL_7]]{{\)\]}} : memref<10xindex> - # CHECK: } - # CHECK: return - # CHECK: } - @func.FuncOp.from_py_func(index_type, index_type, index_type, memref_t) - def range_loop_2(lb, ub, step, memref_v): - for i in range(0, 10, 1): + memref.store(add, memref_v, [i]) + affine.yield_([]) + + # CHECK-LABEL: func.func @range_loop_4( + # CHECK-SAME: %[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: memref<10xindex>) { + # CHECK: affine.for %[[VAL_3:.*]] = 0 to 10 { + # CHECK: %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_3]] : index + # CHECK: memref.store %[[VAL_4]], %[[VAL_2]]{{\[}}%[[VAL_3]]] : memref<10xindex> + # CHECK: } + # CHECK: return + # CHECK: } + @func.FuncOp.from_py_func(T.index(), T.index(), memref_t) + def range_loop_4(lb, ub, memref_v): + for i in range(0, 10, step=1): add = arith.addi(i, i) - s0 = AffineSymbolExpr.get(0) - map = AffineMap.get(0, 1, [s0]) - affine.store(add, memref_v, [i], map=map) - affine.AffineYieldOp([]) - - # CHECK: func.func @range_loop_3(%[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: index, %[[VAL_3:.*]]: memref<10xindex>) { - # CHECK: %[[VAL_4:.*]] = arith.constant 0 : index - # CHECK: affine.for %[[VAL_6:.*]] = %[[VAL_4]] to %[[VAL_1]] { - # CHECK: %[[VAL_7:.*]] = arith.addi %[[VAL_6]], %[[VAL_6]] : index - # CHECK: affine.store %[[VAL_7]], %[[VAL_3]]{{\[symbol\(}}%[[VAL_6]]{{\)\]}} : memref<10xindex> - # CHECK: } - # CHECK: return - # CHECK: } - @func.FuncOp.from_py_func(index_type, index_type, index_type, memref_t) - def range_loop_3(lb, ub, step, memref_v): - for i in range(0, ub, 1): + memref.store(add, memref_v, [i]) + affine.yield_([]) + + # CHECK-LABEL: func.func @range_loop_8( + # CHECK-SAME: %[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: memref<10xindex>) { + # CHECK: %[[VAL_3:.*]] = affine.for %[[VAL_4:.*]] = 0 to 10 iter_args(%[[VAL_5:.*]] = %[[VAL_2]]) -> (memref<10xindex>) { + # CHECK: %[[VAL_6:.*]] = arith.addi %[[VAL_4]], %[[VAL_4]] : index + # CHECK: memref.store %[[VAL_6]], %[[VAL_5]]{{\[}}%[[VAL_4]]] : memref<10xindex> + # CHECK: affine.yield %[[VAL_5]] : memref<10xindex> + # CHECK: } + # CHECK: return + # CHECK: } + @func.FuncOp.from_py_func(T.index(), T.index(), memref_t) + def range_loop_8(lb, ub, memref_v): + for i, it in range(0, 10, iter_args=[memref_v]): add = arith.addi(i, i) - s0 = AffineSymbolExpr.get(0) - map = AffineMap.get(0, 1, [s0]) - affine.store(add, memref_v, [i], map=map) - affine.AffineYieldOp([]) + memref.store(add, it, [i]) + affine.yield_([it]) diff --git a/mlir/unittests/Pass/CMakeLists.txt b/mlir/unittests/Pass/CMakeLists.txt index 65f07741238659df2f319497fdeec91f21cffc1d..802b3bbc6c63527a3ad967b9dd52b2327541a9e2 100644 --- a/mlir/unittests/Pass/CMakeLists.txt +++ b/mlir/unittests/Pass/CMakeLists.txt @@ -5,5 +5,6 @@ add_mlir_unittest(MLIRPassTests ) target_link_libraries(MLIRPassTests PRIVATE + MLIRDebug MLIRFuncDialect MLIRPass) diff --git a/mlir/unittests/Pass/PassManagerTest.cpp b/mlir/unittests/Pass/PassManagerTest.cpp index 9a30f64eaabc2930f94853b74a7a1db613b4a672..7ceed3bb3bc3bd21fa025f13f977998295bc96fe 100644 --- a/mlir/unittests/Pass/PassManagerTest.cpp +++ b/mlir/unittests/Pass/PassManagerTest.cpp @@ -7,6 +7,8 @@ //===----------------------------------------------------------------------===// #include "mlir/Pass/PassManager.h" +#include "mlir/Debug/BreakpointManagers/TagBreakpointManager.h" +#include "mlir/Debug/ExecutionContext.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/IR/Builders.h" #include "mlir/IR/BuiltinOps.h" @@ -86,6 +88,104 @@ TEST(PassManagerTest, OpSpecificAnalysis) { } } +/// Simple pass to annotate a func::FuncOp with a single attribute `didProcess`. +struct AddAttrFunctionPass + : public PassWrapper> { + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(AddAttrFunctionPass) + + void runOnOperation() override { + func::FuncOp op = getOperation(); + Builder builder(op->getParentOfType()); + if (op->hasAttr("didProcess")) + op->setAttr("didProcessAgain", builder.getUnitAttr()); + + // We always want to set this one. + op->setAttr("didProcess", builder.getUnitAttr()); + } +}; + +/// Simple pass to annotate a func::FuncOp with a single attribute +/// `didProcess2`. +struct AddSecondAttrFunctionPass + : public PassWrapper> { + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(AddSecondAttrFunctionPass) + + void runOnOperation() override { + func::FuncOp op = getOperation(); + Builder builder(op->getParentOfType()); + op->setAttr("didProcess2", builder.getUnitAttr()); + } +}; + +TEST(PassManagerTest, ExecutionAction) { + MLIRContext context; + context.loadDialect(); + Builder builder(&context); + + // Create a module with 2 functions. + OwningOpRef module(ModuleOp::create(UnknownLoc::get(&context))); + auto f = + func::FuncOp::create(builder.getUnknownLoc(), "process_me_once", + builder.getFunctionType(std::nullopt, std::nullopt)); + f.setPrivate(); + module->push_back(f); + + // Instantiate our passes. + auto pm = PassManager::on(&context); + auto pass = std::make_unique(); + auto *passPtr = pass.get(); + pm.addNestedPass(std::move(pass)); + pm.addNestedPass(std::make_unique()); + // Duplicate the first pass to ensure that we *only* run the *first* pass, not + // all instances of this pass kind. Notice that this pass (and the test as a + // whole) are built to ensure that we can run just a single pass out of a + // pipeline that may contain duplicates. + pm.addNestedPass(std::make_unique()); + + // Use the action manager to only hit the first pass, not the second one. + auto onBreakpoint = [&](const tracing::ActionActiveStack *backtrace) + -> tracing::ExecutionContext::Control { + // Not a PassExecutionAction, apply the action. + auto *passExec = dyn_cast(&backtrace->getAction()); + if (!passExec) + return tracing::ExecutionContext::Next; + + // If this isn't a function, apply the action. + if (!isa(passExec->getOp())) + return tracing::ExecutionContext::Next; + + // Only apply the first function pass. Not all instances of the first pass, + // only the first pass. + if (passExec->getPass().getThreadingSiblingOrThis() == passPtr) + return tracing::ExecutionContext::Next; + + // Do not apply any other passes in the pass manager. + return tracing::ExecutionContext::Skip; + }; + + // Set up our breakpoint manager. + tracing::TagBreakpointManager simpleManager; + tracing::ExecutionContext executionCtx(onBreakpoint); + executionCtx.addBreakpointManager(&simpleManager); + simpleManager.addBreakpoint(PassExecutionAction::tag); + + // Register the execution context in the MLIRContext. + context.registerActionHandler(executionCtx); + + // Run the pass manager, expecting our handler to be called. + LogicalResult result = pm.run(module.get()); + EXPECT_TRUE(succeeded(result)); + + // Verify that each function got annotated with `didProcess` and *not* + // `didProcess2`. + for (func::FuncOp func : module->getOps()) { + ASSERT_TRUE(func->getDiscardableAttr("didProcess")); + ASSERT_FALSE(func->getDiscardableAttr("didProcess2")); + ASSERT_FALSE(func->getDiscardableAttr("didProcessAgain")); + } +} + namespace { struct InvalidPass : Pass { MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(InvalidPass) diff --git a/openmp/libomptarget/CMakeLists.txt b/openmp/libomptarget/CMakeLists.txt index 115189a28ce1601bdc5133ac5ef4b71256177b6e..66925ccbe03054ec3f4f507001faf074bb41b37a 100644 --- a/openmp/libomptarget/CMakeLists.txt +++ b/openmp/libomptarget/CMakeLists.txt @@ -24,7 +24,7 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${LIBOMPTARGET_LIBRARY_DIR}) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${LIBOMPTARGET_LIBRARY_DIR}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${LIBOMPTARGET_LIBRARY_DIR}) -if(LLVM_LIBRARY_OUTPUT_INTDIR) +if(NOT LLVM_LIBRARY_OUTPUT_INTDIR) set(LIBOMPTARGET_INTDIR ${LIBOMPTARGET_LIBRARY_DIR}) else() set(LIBOMPTARGET_INTDIR ${LLVM_LIBRARY_OUTPUT_INTDIR}) @@ -75,6 +75,25 @@ if(LIBOMPTARGET_ENABLE_DEBUG) add_definitions(-DOMPTARGET_DEBUG) endif() +# No exceptions and no RTTI, except if requested. +set(offload_compile_flags -fno-exceptions) +if(NOT LLVM_ENABLE_RTTI) + set(offload_compile_flags ${offload_compile_flags} -fno-rtti) +endif() + +# If LTO is not explicitly disabled we check if we can enable it and do so. +set(LIBOMPTARGET_USE_LTO TRUE CACHE BOOL "Use LTO for the offload runtimes if available") +if (LIBOMPTARGET_USE_LTO) + include(CheckIPOSupported) + check_ipo_supported(RESULT use_lto OUTPUT output) + if(use_lto) + set(offload_compile_flags ${offload_compile_flags} -flto) + set(offload_link_flags ${offload_link_flags} -flto) + else() + message(WARNING "LTO is not supported: ${output}") + endif() +endif() + # OMPT support for libomptarget # Follow host OMPT support and check if host support has been requested. # LIBOMP_HAVE_OMPT_SUPPORT indicates whether host OMPT support has been implemented. diff --git a/openmp/libomptarget/DeviceRTL/CMakeLists.txt b/openmp/libomptarget/DeviceRTL/CMakeLists.txt index 104b65020daf2f4c1f1ff454958c8a2a6e1deb64..1ce3e1e40a80ab480a4576818420cc370d52fb5c 100644 --- a/openmp/libomptarget/DeviceRTL/CMakeLists.txt +++ b/openmp/libomptarget/DeviceRTL/CMakeLists.txt @@ -302,7 +302,7 @@ endforeach() # Archive all the object files generated above into a static library add_library(omptarget.devicertl STATIC) set_target_properties(omptarget.devicertl PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY ${LIBOMPTARGET_LLVM_LIBRARY_INTDIR} + ARCHIVE_OUTPUT_DIRECTORY "${LIBOMPTARGET_LLVM_LIBRARY_INTDIR}" LINKER_LANGUAGE CXX ) target_link_libraries(omptarget.devicertl PRIVATE omptarget.devicertl.all_objs) diff --git a/openmp/libomptarget/include/PluginManager.h b/openmp/libomptarget/include/PluginManager.h index 6af99efa32a0cab4fdc7400501dd608eb6b00a48..0b0974709b525660c1a310325d42030ebab356bc 100644 --- a/openmp/libomptarget/include/PluginManager.h +++ b/openmp/libomptarget/include/PluginManager.h @@ -150,6 +150,24 @@ struct PluginManager { HostPtrToTableMapTy HostPtrToTableMap; std::mutex TblMapMtx; ///< For HostPtrToTableMap + // Work around for plugins that call dlopen on shared libraries that call + // tgt_register_lib during their initialisation. Stash the pointers in a + // vector until the plugins are all initialised and then register them. + bool delayRegisterLib(__tgt_bin_desc *Desc) { + if (RTLsLoaded) + return false; + DelayedBinDesc.push_back(Desc); + return true; + } + + void registerDelayedLibraries() { + // Only called by libomptarget constructor + RTLsLoaded = true; + for (auto *Desc : DelayedBinDesc) + __tgt_register_lib(Desc); + DelayedBinDesc.clear(); + } + /// Return the number of usable devices. int getNumDevices() { return getExclusiveDevicesAccessor()->size(); } @@ -178,6 +196,9 @@ struct PluginManager { void addRequirements(int64_t Flags) { Requirements.addRequirements(Flags); } private: + bool RTLsLoaded = false; + llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc; + // List of all plugin adaptors, in use or not. llvm::SmallVector> PluginAdaptors; diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt index 5b332ed3d2f41735f962a2573776d59fec10a1a3..8ae3ff2a6d29164c15b92fa45b68295a8123a250 100644 --- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt @@ -88,6 +88,9 @@ target_compile_definitions(PluginCommon PRIVATE DEBUG_PREFIX="PluginInterface" ) +target_compile_options(PluginCommon PUBLIC ${offload_compile_flags}) +target_link_options(PluginCommon PUBLIC ${offload_link_flags}) + target_include_directories(PluginCommon PRIVATE ${LIBOMPTARGET_INCLUDE_DIR} diff --git a/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp b/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp index a3d16d3a5bcff2d376f8ce076ec9a9209b2103f6..0a19148ca4ec6b24cb469ff94e2a6a2effbbc30f 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp @@ -16,6 +16,10 @@ #include "Shared/Utils.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/Support/Error.h" + +#include #include using namespace llvm; @@ -53,9 +57,15 @@ Error GenericGlobalHandlerTy::getGlobalMetadataFromELF( const ELF64LE::Shdr &Section, GlobalTy &ImageGlobal) { // The global's address is computed as the image begin + the ELF section - // offset + the ELF symbol value. - ImageGlobal.setPtr(advanceVoidPtr( - Image.getStart(), Section.sh_offset - Section.sh_addr + Symbol.st_value)); + // offset + the ELF symbol value except for NOBITS sections that, as the name + // suggests, have no bits in the image. We still record the size and use + // nullptr to indicate there is no location. + if (Section.sh_type == ELF::SHT_NOBITS) + ImageGlobal.setPtr(nullptr); + else + ImageGlobal.setPtr( + advanceVoidPtr(Image.getStart(), + Section.sh_offset - Section.sh_addr + Symbol.st_value)); // Set the global's size. ImageGlobal.setSize(Symbol.st_size); @@ -170,12 +180,21 @@ Error GenericGlobalHandlerTy::readGlobalFromImage(GenericDeviceTy &Device, "%u bytes in the ELF image but %u bytes on the host", HostGlobal.getName().data(), ImageGlobal.getSize(), HostGlobal.getSize()); + if (ImageGlobal.getPtr() == nullptr) + return Plugin::error("Transfer impossible because global symbol '%s' has " + "no representation in the image (NOBITS sections)", + HostGlobal.getName().data()); DP("Global symbol '%s' was found in the ELF image and %u bytes will copied " "from %p to %p.\n", HostGlobal.getName().data(), HostGlobal.getSize(), ImageGlobal.getPtr(), HostGlobal.getPtr()); + assert(Image.getStart() <= ImageGlobal.getPtr() && + advanceVoidPtr(ImageGlobal.getPtr(), ImageGlobal.getSize()) < + advanceVoidPtr(Image.getStart(), Image.getSize()) && + "Attempting to read outside the image!"); + // Perform the copy from the image to the host memory. std::memcpy(HostGlobal.getPtr(), ImageGlobal.getPtr(), HostGlobal.getSize()); diff --git a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp index 912e3d2c479b3021760d03f5ab9cb7927e759b54..3c7d1ca8998787cecdf8f1c485eb2f6739b8205a 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp @@ -785,9 +785,14 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) { GenericGlobalHandlerTy &GHandler = Plugin.getGlobalHandler(); for (auto *Image : LoadedImages) { DeviceMemoryPoolTrackingTy ImageDeviceMemoryPoolTracking = {0, 0, ~0U, 0}; - if (!GHandler.isSymbolInImage(*this, *Image, - "__omp_rtl_device_memory_pool_tracker")) + GlobalTy TrackerGlobal("__omp_rtl_device_memory_pool_tracker", + sizeof(DeviceMemoryPoolTrackingTy), + &ImageDeviceMemoryPoolTracking); + if (auto Err = + GHandler.readGlobalFromDevice(*this, *Image, TrackerGlobal)) { + consumeError(std::move(Err)); continue; + } DeviceMemoryPoolTracking.combine(ImageDeviceMemoryPoolTracking); } @@ -968,16 +973,16 @@ Error GenericDeviceTy::setupDeviceMemoryPool(GenericPluginTy &Plugin, } // Create the metainfo of the device environment global. - GlobalTy TrackerGlobal("__omp_rtl_device_memory_pool_tracker", - sizeof(DeviceMemoryPoolTrackingTy), - &DeviceMemoryPoolTracking); GenericGlobalHandlerTy &GHandler = Plugin.getGlobalHandler(); - if (auto Err = GHandler.readGlobalFromImage(*this, Image, TrackerGlobal)) { - [[maybe_unused]] std::string ErrStr = toString(std::move(Err)); - DP("Avoid the memory pool: %s.\n", ErrStr.c_str()); + if (!GHandler.isSymbolInImage(*this, Image, + "__omp_rtl_device_memory_pool_tracker")) { + DP("Skip the memory pool as there is no tracker symbol in the image."); return Error::success(); } + GlobalTy TrackerGlobal("__omp_rtl_device_memory_pool_tracker", + sizeof(DeviceMemoryPoolTrackingTy), + &DeviceMemoryPoolTracking); if (auto Err = GHandler.writeGlobalToDevice(*this, Image, TrackerGlobal)) return Err; diff --git a/openmp/libomptarget/src/CMakeLists.txt b/openmp/libomptarget/src/CMakeLists.txt index 7c07c61142bace60b8e7497bc4728e300b3257fa..429a670be84957a49cf31be9ca2850253d893496 100644 --- a/openmp/libomptarget/src/CMakeLists.txt +++ b/openmp/libomptarget/src/CMakeLists.txt @@ -55,6 +55,9 @@ target_compile_definitions(omptarget PRIVATE DEBUG_PREFIX="omptarget" ) +target_compile_options(omptarget PUBLIC ${offload_compile_flags}) +target_link_options(omptarget PUBLIC ${offload_link_flags}) + macro(check_plugin_target target) if (TARGET omptarget.rtl.${target}) list(APPEND LIBOMPTARGET_PLUGINS_TO_LOAD ${target}) diff --git a/openmp/libomptarget/src/PluginManager.cpp b/openmp/libomptarget/src/PluginManager.cpp index f93e88e9e27476526c3abe050ee71cf80a93de18..16e04dde923d9e3e735c6138ac0dd6c3ae842a7c 100644 --- a/openmp/libomptarget/src/PluginManager.cpp +++ b/openmp/libomptarget/src/PluginManager.cpp @@ -61,7 +61,7 @@ Error PluginAdaptorTy::init() { return createStringError(inconvertibleErrorCode(), \ "Invalid plugin as necessary interface function " \ "(%s) was not found.\n", \ - NAME); \ + std::string(#NAME).c_str()); \ } #include "Shared/PluginAPI.inc" diff --git a/openmp/libomptarget/src/interface.cpp b/openmp/libomptarget/src/interface.cpp index 777dc07943aa634ef0ca5bda340aab2d395f7c05..d92f40ce1d14e91ce385c1c2d572504299b621c4 100644 --- a/openmp/libomptarget/src/interface.cpp +++ b/openmp/libomptarget/src/interface.cpp @@ -46,6 +46,9 @@ EXTERN void __tgt_register_requires(int64_t Flags) { /// adds a target shared library to the target execution image EXTERN void __tgt_register_lib(__tgt_bin_desc *Desc) { TIMESCOPE(); + if (PM->delayRegisterLib(Desc)) + return; + PM->registerLib(Desc); } diff --git a/openmp/libomptarget/src/rtl.cpp b/openmp/libomptarget/src/rtl.cpp index 27db7034d895658188d709bdda6159ff7ec0ce9f..5eb1c553df4916f16302d04b6581367ad20df99d 100644 --- a/openmp/libomptarget/src/rtl.cpp +++ b/openmp/libomptarget/src/rtl.cpp @@ -51,6 +51,7 @@ __attribute__((constructor(101))) void init() { PM->init(); Profiler::get(); + PM->registerDelayedLibraries(); } __attribute__((destructor(101))) void deinit() { diff --git a/openmp/libomptarget/test/Inputs/empty.c b/openmp/libomptarget/test/Inputs/empty.c deleted file mode 100644 index 8b137891791fe96927ad78e64b0aad7bded08bdc..0000000000000000000000000000000000000000 --- a/openmp/libomptarget/test/Inputs/empty.c +++ /dev/null @@ -1 +0,0 @@ - diff --git a/openmp/libomptarget/test/offloading/back2back_distribute.c b/openmp/libomptarget/test/offloading/back2back_distribute.c index 750e39061974a3be652eae09aaf332c2b4ca32ac..63cabd0c66788b5931411f9e193692410f588811 100644 --- a/openmp/libomptarget/test/offloading/back2back_distribute.c +++ b/openmp/libomptarget/test/offloading/back2back_distribute.c @@ -15,12 +15,12 @@ void reset_input(double *a, double *a_h, double *b, double *c) { } int main(int argc, char *argv[]) { - double * a = (double *) malloc(MAX_N * sizeof(double)); - double * a_h = (double *) malloc(MAX_N * sizeof(double)); - double * d = (double *) malloc(MAX_N * sizeof(double)); - double * d_h = (double *) malloc(MAX_N * sizeof(double)); - double * b = (double *) malloc(MAX_N * sizeof(double)); - double * c = (double *) malloc(MAX_N * sizeof(double)); + double *a = (double *)calloc(MAX_N, sizeof(double)); + double *a_h = (double *)calloc(MAX_N, sizeof(double)); + double *d = (double *)calloc(MAX_N, sizeof(double)); + double *d_h = (double *)calloc(MAX_N, sizeof(double)); + double *b = (double *)calloc(MAX_N, sizeof(double)); + double *c = (double *)calloc(MAX_N, sizeof(double)); #pragma omp target enter data map(to:a[:MAX_N],b[:MAX_N],c[:MAX_N],d[:MAX_N]) diff --git a/openmp/libomptarget/test/offloading/barrier_fence.c b/openmp/libomptarget/test/offloading/barrier_fence.c index a0b672fb1a84a294e373904050ba11510e4fed59..5d1096478ed9e965c763b0ef99917122162fc0c2 100644 --- a/openmp/libomptarget/test/offloading/barrier_fence.c +++ b/openmp/libomptarget/test/offloading/barrier_fence.c @@ -7,8 +7,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: amdgcn-amd-amdhsa -// UNSUPPORTED: amdgcn-amd-amdhsa-LTO #include #include diff --git a/openmp/libomptarget/test/offloading/bug60119.c b/openmp/libomptarget/test/offloading/bug60119.c deleted file mode 100644 index e32f1ccd598812ebc0f85442c707f1409ae5c16e..0000000000000000000000000000000000000000 --- a/openmp/libomptarget/test/offloading/bug60119.c +++ /dev/null @@ -1,6 +0,0 @@ -// RUN: %clang-generic -fPIC -shared %S/../Inputs/empty.c -o %T/liba.so -// RUN: %clang-generic -fPIC -shared %S/../Inputs/empty.c -o %T/libb.so -// RUN: %clang-generic -rpath %T -L %T -l a -l b %s -o %t -// RUN: %t - -int main() {} diff --git a/openmp/libomptarget/test/offloading/bug74582.c b/openmp/libomptarget/test/offloading/bug74582.c new file mode 100644 index 0000000000000000000000000000000000000000..c6a283bb936914370e306ee5485d366a0c966017 --- /dev/null +++ b/openmp/libomptarget/test/offloading/bug74582.c @@ -0,0 +1,13 @@ +// RUN: %libomptarget-compile-generic && %libomptarget-run-generic +// RUN: %libomptarget-compileopt-generic && %libomptarget-run-generic + +// Verify we do not read bits in the image that are not there (nobits section). + +#pragma omp begin declare target +char BigUninitializedBuffer[4096 * 64] __attribute__((loader_uninitialized)); +#pragma omp end declare target + +int main() { +#pragma omp target + {} +} diff --git a/openmp/libomptarget/test/offloading/shared_lib_fp_mapping.c b/openmp/libomptarget/test/offloading/shared_lib_fp_mapping.c index a98a84ccceaab9ea6d9d79cbcde2e3eab4e61382..ffb4bc90d4e4f530fabcad3eabff1133178c9b26 100644 --- a/openmp/libomptarget/test/offloading/shared_lib_fp_mapping.c +++ b/openmp/libomptarget/test/offloading/shared_lib_fp_mapping.c @@ -1,6 +1,6 @@ // clang-format off -// RUN: %clang-generic -fPIC -shared %S/../Inputs/declare_indirect_func.c -o %T/liba.so -fopenmp-version=51 -// RUN: %libomptarget-compile-generic -rpath %T -L %T -l a -o %t -fopenmp-version=51 +// RUN: %clang-generic -fPIC -shared %S/../Inputs/declare_indirect_func.c -o %T/libslfm.so -fopenmp-version=51 +// RUN: %libomptarget-compile-generic -rpath %T -L %T -l slfm -o %t -fopenmp-version=51 // RUN: env LIBOMPTARGET_INFO=32 %t 2>&1 | %fcheck-generic // clang-format on diff --git a/openmp/libomptarget/test/offloading/target_map_for_member_data.cpp b/openmp/libomptarget/test/offloading/target_map_for_member_data.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7fba71be6e662d1944ede94288d79d2be4fee35b --- /dev/null +++ b/openmp/libomptarget/test/offloading/target_map_for_member_data.cpp @@ -0,0 +1,25 @@ +// clang-format off +// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-generic 2>&1 | %fcheck-generic +// clang-format on + +// REQUIRES: libomptarget-debug + +struct DataTy { + float a; + float b[3]; +}; + +int main(int argc, char **argv) { + DataTy D; +#pragma omp target map(D.a) map(D.b[ : 2]) + { + D.a = 0; + D.b[0] = 1; + } + return 0; +} +// clang-format off +// CHECK: omptarget --> Entry 0: Base=[[DAT_HST_PTR_BASE:0x.*]], Begin=[[DAT_HST_PTR_BASE]], Size=12 +// CHECK: omptarget --> Entry 1: Base=[[DAT_HST_PTR_BASE]], Begin=[[DAT_HST_PTR_BASE]], Size=4, +// CHECK: omptarget --> Entry 2: Base=[[DAT_HST_PTR_BASE]], Begin=[[DATUM_HST_PTR_BASE:0x.*]], Size=8, +// clang-format on diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp index 72da0f79865dfb1f6ec61db00623eab4caf7b33c..fdce932fd6d6bd455b842089a21c92faf3222b73 100644 --- a/openmp/runtime/src/z_Linux_util.cpp +++ b/openmp/runtime/src/z_Linux_util.cpp @@ -72,7 +72,7 @@ struct kmp_sys_timer { struct timespec start; }; -#if KMP_OS_SOLARIS +#ifndef TIMEVAL_TO_TIMESPEC // Convert timeval to timespec. #define TIMEVAL_TO_TIMESPEC(tv, ts) \ do { \ diff --git a/polly/lib/Analysis/ScopBuilder.cpp b/polly/lib/Analysis/ScopBuilder.cpp index c34413812d9464e22698ff476c1a72c554daa999..0af0f6915b145851a45001d84af3d9591c3835fe 100644 --- a/polly/lib/Analysis/ScopBuilder.cpp +++ b/polly/lib/Analysis/ScopBuilder.cpp @@ -2510,6 +2510,48 @@ static MemoryAccess::ReductionType getReductionType(const BinaryOperator *BinOp, } } +/// True if @p AllAccs intersects with @p MemAccs execpt @p LoadMA and @p +/// StoreMA +bool hasIntersectingAccesses(isl::set AllAccs, MemoryAccess *LoadMA, + MemoryAccess *StoreMA, isl::set Domain, + SmallVector &MemAccs) { + bool HasIntersectingAccs = false; + for (MemoryAccess *MA : MemAccs) { + if (MA == LoadMA || MA == StoreMA) + continue; + + isl::map AccRel = MA->getAccessRelation().intersect_domain(Domain); + isl::set Accs = AccRel.range(); + + if (AllAccs.has_equal_space(Accs)) { + isl::set OverlapAccs = Accs.intersect(AllAccs); + bool DoesIntersect = !OverlapAccs.is_empty(); + HasIntersectingAccs |= DoesIntersect; + } + } + return HasIntersectingAccs; +} + +/// Test if the accesses of @p LoadMA and @p StoreMA can form a reduction +bool checkCandidatePairAccesses(MemoryAccess *LoadMA, MemoryAccess *StoreMA, + isl::set Domain, + SmallVector &MemAccs) { + isl::map LoadAccs = LoadMA->getAccessRelation(); + isl::map StoreAccs = StoreMA->getAccessRelation(); + + // Skip those with obviously unequal base addresses. + bool Valid = LoadAccs.has_equal_space(StoreAccs); + + // And check if the remaining for overlap with other memory accesses. + if (Valid) { + isl::map AllAccsRel = LoadAccs.unite(StoreAccs); + AllAccsRel = AllAccsRel.intersect_domain(Domain); + isl::set AllAccs = AllAccsRel.range(); + Valid = !hasIntersectingAccesses(AllAccs, LoadMA, StoreMA, Domain, MemAccs); + } + return Valid; +} + void ScopBuilder::checkForReductions(ScopStmt &Stmt) { SmallVector Loads; SmallVector, 4> Candidates; @@ -2528,34 +2570,10 @@ void ScopBuilder::checkForReductions(ScopStmt &Stmt) { // Then check each possible candidate pair. for (const auto &CandidatePair : Candidates) { - bool Valid = true; - isl::map LoadAccs = CandidatePair.first->getAccessRelation(); - isl::map StoreAccs = CandidatePair.second->getAccessRelation(); - - // Skip those with obviously unequal base addresses. - if (!LoadAccs.has_equal_space(StoreAccs)) { - continue; - } - - // And check if the remaining for overlap with other memory accesses. - isl::map AllAccsRel = LoadAccs.unite(StoreAccs); - AllAccsRel = AllAccsRel.intersect_domain(Stmt.getDomain()); - isl::set AllAccs = AllAccsRel.range(); - - for (MemoryAccess *MA : Stmt) { - if (MA == CandidatePair.first || MA == CandidatePair.second) - continue; - - isl::map AccRel = - MA->getAccessRelation().intersect_domain(Stmt.getDomain()); - isl::set Accs = AccRel.range(); - - if (AllAccs.has_equal_space(Accs)) { - isl::set OverlapAccs = Accs.intersect(AllAccs); - Valid = Valid && OverlapAccs.is_empty(); - } - } - + MemoryAccess *LoadMA = CandidatePair.first; + MemoryAccess *StoreMA = CandidatePair.second; + bool Valid = checkCandidatePairAccesses(LoadMA, StoreMA, Stmt.getDomain(), + Stmt.MemAccs); if (!Valid) continue; @@ -2566,8 +2584,8 @@ void ScopBuilder::checkForReductions(ScopStmt &Stmt) { // If no overlapping access was found we mark the load and store as // reduction like. - CandidatePair.first->markAsReductionLike(RT); - CandidatePair.second->markAsReductionLike(RT); + LoadMA->markAsReductionLike(RT); + StoreMA->markAsReductionLike(RT); } } diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel index d3c7443748d9167f9635eb504be7e4d533766add..477e59e366d18e21d6da491ffd910ef15f1339f0 100644 --- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel @@ -1934,10 +1934,12 @@ llvm_target_lib_list = [lib for lib in [ "name": "BPF", "short_name": "BPF", "tbl_outs": [ + ("-gen-register-bank", "lib/Target/BPF/BPFGenRegisterBank.inc"), ("-gen-asm-writer", "lib/Target/BPF/BPFGenAsmWriter.inc"), ("-gen-asm-matcher", "lib/Target/BPF/BPFGenAsmMatcher.inc"), ("-gen-callingconv", "lib/Target/BPF/BPFGenCallingConv.inc"), ("-gen-dag-isel", "lib/Target/BPF/BPFGenDAGISel.inc"), + ("-gen-global-isel", "lib/Target/BPF/BPFGenGlobalISel.inc"), ("-gen-disassembler", "lib/Target/BPF/BPFGenDisassemblerTables.inc"), ("-gen-emitter", "lib/Target/BPF/BPFGenMCCodeEmitter.inc"), ("-gen-instr-info", "lib/Target/BPF/BPFGenInstrInfo.inc"), @@ -2237,10 +2239,10 @@ gentbl( name = "riscv_isel_target_gen", strip_include_prefix = "lib/Target/RISCV", tbl_outs = [ - ("-gen-global-isel", "lib/Target/RISCV/RISCVGenGlobalISel.inc"), - ("-gen-global-isel-combiner -combiners=RISCVO0PreLegalizerCombiner", "lib/Target/RISCV/RISCVGenO0PreLegalizeGICombiner.inc"), - ("-gen-global-isel-combiner -combiners=RISCVPostLegalizerCombiner", "lib/Target/RISCV/RISCVGenPostLegalizeGICombiner.inc"), - ("-gen-global-isel-combiner -combiners=RISCVPreLegalizerCombiner", "lib/Target/RISCV/RISCVGenPreLegalizeGICombiner.inc"), + ("-gen-global-isel", "lib/Target/RISCV/RISCVGenGlobalISel.inc"), + ("-gen-global-isel-combiner -combiners=RISCVO0PreLegalizerCombiner", "lib/Target/RISCV/RISCVGenO0PreLegalizeGICombiner.inc"), + ("-gen-global-isel-combiner -combiners=RISCVPostLegalizerCombiner", "lib/Target/RISCV/RISCVGenPostLegalizeGICombiner.inc"), + ("-gen-global-isel-combiner -combiners=RISCVPreLegalizerCombiner", "lib/Target/RISCV/RISCVGenPreLegalizeGICombiner.inc"), ], tblgen = ":llvm-tblgen", td_file = "lib/Target/RISCV/RISCVGISel.td", @@ -2341,10 +2343,10 @@ gentbl( deps = [ ":BinaryFormat", ":CodeGenTypes", + ":Core", ":DebugInfoCodeView", ":MC", ":MCDisassembler", - ":Core", ":Support", ":Target", ":config", @@ -2550,23 +2552,23 @@ cc_library( ":AggressiveInstCombine", ":Analysis", ":CodeGen", - ":common_transforms", - ":config", ":Core", ":Coroutines", ":HipStdPar", - ":InstCombine", - ":Instrumentation", ":IPO", ":IRPrinter", + ":InstCombine", + ":Instrumentation", ":MLPolicies", ":ObjCARC", - ":pass_registry_def", ":Scalar", ":Support", ":Target", ":TransformUtils", ":Vectorize", + ":common_transforms", + ":config", + ":pass_registry_def", ], ) @@ -2802,20 +2804,19 @@ cc_library( ]), hdrs = glob([ "include/llvm/ExecutionEngine/Orc/Debugging/*.h", - ]), + ]) + ["include/llvm-c/LLJITUtils.h"], copts = llvm_copts, deps = [ ":BinaryFormat", ":DebugInfo", - ":OrcShared", - ":OrcJIT", ":JITLink", + ":OrcJIT", + ":OrcShared", ":Support", ":TargetParser", ], ) - cc_library( name = "OrcTargetProcess", srcs = glob([ @@ -3117,9 +3118,9 @@ cc_library( ":MC", ":MCA", ":MCDisassembler", - ":OrcJIT", ":Object", ":ObjectYAML", + ":OrcJIT", ":Support", ":Target", ":config", @@ -3249,9 +3250,9 @@ cc_binary( ":Interpreter", ":MCJIT", ":Object", + ":OrcDebugging", ":OrcJIT", ":OrcTargetProcess", - ":OrcDebugging", ":Support", ":TargetParser", ":config", @@ -4703,7 +4704,7 @@ cc_binary( srcs = glob([ "tools/llvm-symbolizer/*.cpp", "tools/llvm-symbolizer/*.h", - ]) + [ "llvm-symbolizer-driver.cpp"], + ]) + ["llvm-symbolizer-driver.cpp"], copts = llvm_copts, stamp = 0, deps = [ @@ -4816,7 +4817,6 @@ expand_template( template = "cmake/modules/llvm-driver-template.cpp.in", ) - cc_binary( name = "sancov", srcs = glob([ diff --git a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel index 42b417548d94c254a890e1964018dcd56eab72d1..d2bcfc14ff75cdba4f05dd7eaf054176dafe91db 100644 --- a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel @@ -309,6 +309,7 @@ cc_test( "//llvm:JITLink", "//llvm:MC", "//llvm:Object", + "//llvm:OrcDebugging", "//llvm:OrcJIT", "//llvm:OrcShared", "//llvm:OrcTargetProcess", diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 7b64a4956e1ec49f4bc83cac1fcbd18b2e5a3585..c831df20349e2cc595517ef55f5a41aaa5ff4999 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -3994,7 +3994,6 @@ cc_library( deps = [ ":ArithDialect", ":ArithUtils", - ":BufferizationDialect", ":ControlFlowDialect", ":ControlFlowInterfaces", ":DestinationStyleOpInterface", diff --git a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel index c4881383b5c59aa9a6752cd9107ee5558fdd1652..d20bd403e8f8dab9b76e5e326e4d61c2c8736689 100644 --- a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel @@ -101,6 +101,7 @@ cc_test( "//llvm:Support", "//llvm:TestingSupport", "//mlir:Analysis", + "//mlir:Debug", "//mlir:FuncDialect", "//mlir:IR", "//mlir:Pass",

().g()), ""); @@ -214,57 +258,64 @@ namespace dr1330 { // dr1330: 4 c++11 static_assert(!noexcept(B().g()), ""); #endif - template int f() throw(typename T::error) { return 0; } // expected-error 1-4{{prior to '::'}} expected-note 0-1{{prior to '::'}} expected-note 0-1{{requested here}} -#if __cplusplus > 201402L - // expected-error@-2 0-1{{C++17}} expected-note@-2 0-1{{noexcept}} -#endif + template int f() throw(typename T::error) { return 0; } // #dr1330-f + // expected-error@#dr1330-f {{type 'int' cannot be used prior to '::' because it has no members}} + // cxx98-note@#dr1330-f-int {{in instantiation of function template specialization 'dr1330::f' requested here}} + // since-cxx11-note@#dr1330-f-int {{in instantiation of exception specification for 'f' requested here}} + // cxx98-14-error@#dr1330-f {{type 'short' cannot be used prior to '::' because it has no members}} + // cxx98-14-note@#dr1330-f-short {{in instantiation of function template specialization 'dr1330::f' requested here}} + // cxx11-14-note@#dr1330-f {{in instantiation of exception specification for 'f' requested here}} + // since-cxx11-error@#dr1330-f {{type 'char' cannot be used prior to '::' because it has no members}} + // since-cxx11-note@#dr1330-f-char {{in instantiation of exception specification for 'f' requested here}} + // since-cxx11-error@#dr1330-f {{type 'float' cannot be used prior to '::' because it has no members}} + // since-cxx11-note@#dr1330-f-float {{in instantiation of exception specification for 'f' requested here}} + // since-cxx17-error@#dr1330-f {{ISO C++17 does not allow dynamic exception specifications}} + // since-cxx17-note@#dr1330-f {{use 'noexcept(false)' instead}} + // An exception-specification is needed even if the function is only used in // an unevaluated operand. - int f1 = sizeof(f()); // expected-note {{instantiation of}} + int f1 = sizeof(f()); // #dr1330-f-int #if __cplusplus >= 201103L - decltype(f()) f2; // expected-note {{instantiation of}} - bool f3 = noexcept(f()); // expected-note {{instantiation of}} + decltype(f()) f2; // #dr1330-f-char + bool f3 = noexcept(f()); /// #dr1330-f-float #endif // In C++17 onwards, substituting explicit template arguments into the // function type substitutes into the exception specification (because it's // part of the type). In earlier languages, we don't notice there's a problem // until we've already started to instantiate. - template int f(); -#if __cplusplus >= 201703L - // expected-error@-2 {{does not refer to a function template}} -#else - // expected-note@-4 {{instantiation of}} -#endif + template int f(); // #dr1330-f-short + // since-cxx17-error@-1 {{explicit instantiation of 'f' does not refer to a function template, variable template, member function, member class, or static data member}} + // since-cxx17-note@#dr1330-f {{candidate template ignored: substitution failure [with T = short]: type 'short' cannot be used prior to '::' because it has no members}} template struct C { - C() throw(typename T::type); // expected-error 1-2{{prior to '::'}} -#if __cplusplus > 201402L - // expected-error@-2 0-1{{C++17}} expected-note@-2 0-1{{noexcept}} -#endif + C() throw(typename T::type); // #dr1330-C + // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}} + // since-cxx17-note@-2 {{use 'noexcept(false)' instead}} + // cxx98-error@#dr1330-C {{type 'void' cannot be used prior to '::' because it has no members}} + // cxx98-note@#dr1330-C-void {{in instantiation of template class 'dr1330::C' requested here}} + // expected-error@#dr1330-C {{type 'int' cannot be used prior to '::' because it has no members}} + // cxx98-note@#dr1330-C-int {{in instantiation of template class 'dr1330::C' requested here}} + // since-cxx11-note@#dr1330-C-int {{in instantiation of exception specification for 'C' requested here}} + // since-cxx11-note@#dr1330-e {{in evaluation of exception specification for 'dr1330::E::E' needed here}} }; - struct D : C {}; // ok -#if __cplusplus < 201103L - // expected-note@-2 {{instantiation of}} -#endif + struct D : C {}; // #dr1330-C-void void f(D &d) { d = d; } // ok - struct E : C {}; // expected-note {{in instantiation of}} -#if __cplusplus >= 201103L - E e; // expected-note {{needed here}} -#endif + struct E : C {}; // #dr1330-C-int + E e; // #dr1330-e } namespace dr1341 { // dr1341: sup P0683R1 #if __cplusplus >= 202002L int a; -const int b = 0; // #dr1341-b-decl +const int b = 0; // #dr1341-b struct S { int x1 : 8 = 42; int x2 : 8 { 42 }; int y1 : true ? 8 : a = 42; int y2 : true ? 8 : b = 42; - // expected-error@-1 {{cannot assign to variable 'b' with const-qualified type 'const int'}} - // expected-note@#dr1341-b-decl {{variable 'b' declared const here}} + // since-cxx20-error@-1 {{cannot assign to variable 'b' with const-qualified type 'const int'}} + // since-cxx20-note@#dr1341-b {{variable 'b' declared const here}} int y3 : (true ? 8 : b) = 42; int z : 1 || new int { 0 }; }; @@ -272,44 +323,66 @@ struct S { } namespace dr1346 { // dr1346: 3.5 - auto a(1); // expected-error 0-1{{extension}} - auto b(1, 2); // expected-error {{multiple expressions}} expected-error 0-1{{extension}} + auto a(1); + // cxx98-error@-1 {{'auto' type specifier is a C++11 extension}} + auto b(1, 2); + // cxx98-error@-1 {{'auto' type specifier is a C++11 extension}} + // expected-error@-2 {{initializer for variable 'b' with type 'auto' contains multiple expressions}} #if __cplusplus >= 201103L - auto c({}); // expected-error {{parenthesized initializer list}} - auto d({1}); // expected-error {{parenthesized initializer list}} - auto e({1, 2}); // expected-error {{parenthesized initializer list}} + auto c({}); + // since-cxx11-error@-1 {{cannot deduce type for variable 'c' with type 'auto' from parenthesized initializer list}} + auto d({1}); + // since-cxx11-error@-1 {{cannot deduce type for variable 'd' with type 'auto' from parenthesized initializer list}} + auto e({1, 2}); + // since-cxx11-error@-1 {{cannot deduce type for variable 'e' with type 'auto' from parenthesized initializer list}} #endif - template void f(Ts ...ts) { // expected-error 0-1{{extension}} - auto x(ts...); // expected-error {{empty}} expected-error 0-1{{extension}} + template void f(Ts ...ts) { + // cxx98-error@-1 {{variadic templates are a C++11 extension}} + auto x(ts...); + // cxx98-error@-1 {{'auto' type specifier is a C++11 extension}} + // expected-error@-2 {{initializer for variable 'x' with type 'auto' is empty}} + // expected-note@#dr1346-f {{in instantiation of function template specialization 'dr1346::f<>' requested here}} } - template void f(); // expected-note {{instantiation}} + template void f(); // #dr1346-f #if __cplusplus >= 201103L void init_capture() { - [a(1)] {} (); // expected-error 0-1{{extension}} - [b(1, 2)] {} (); // expected-error {{multiple expressions}} expected-error 0-1{{extension}} -#if __cplusplus >= 201103L - [c({})] {} (); // expected-error {{parenthesized initializer list}} expected-error 0-1{{extension}} - [d({1})] {} (); // expected-error {{parenthesized initializer list}} expected-error 0-1{{extension}} - [e({1, 2})] {} (); // expected-error {{parenthesized initializer list}} expected-error 0-1{{extension}} -#endif + [a(1)] {} (); + // cxx11-error@-1 {{initialized lambda captures are a C++14 extension}} + [b(1, 2)] {} (); + // cxx11-error@-1 {{initialized lambda captures are a C++14 extension}} + // since-cxx11-error@-2 {{initializer for lambda capture 'b' contains multiple expressions}} + [c({})] {} (); + // cxx11-error@-1 {{initialized lambda captures are a C++14 extension}} + // since-cxx11-error@-2 {{cannot deduce type for lambda capture 'c' from parenthesized initializer list}} + [d({1})] {} (); + // cxx11-error@-1 {{initialized lambda captures are a C++14 extension}} + // since-cxx11-error@-2 {{cannot deduce type for lambda capture 'd' from parenthesized initializer list}} + [e({1, 2})] {} (); + // cxx11-error@-1 {{initialized lambda captures are a C++14 extension}} + // since-cxx11-error@-2 {{cannot deduce type for lambda capture 'e' from parenthesized initializer list}} } #endif } -namespace dr1347 { // dr1347: yes - auto x = 5, *y = &x; // expected-error 0-1{{extension}} - auto z = y, *q = y; // expected-error {{'auto' deduced as 'int *' in declaration of 'z' and deduced as 'int' in declaration of 'q'}} expected-error 0-1{{extension}} +namespace dr1347 { // dr1347: 3.1 + auto x = 5, *y = &x; + // cxx98-error@-1 {{'auto' type specifier is a C++11 extension}} + auto z = y, *q = y; + // cxx98-error@-1 {{'auto' type specifier is a C++11 extension}} + // expected-error@-2 {{'auto' deduced as 'int *' in declaration of 'z' and deduced as 'int' in declaration of 'q'}} #if __cplusplus >= 201103L - auto a = 5, b = {1, 2}; // expected-error {{'auto' deduced as 'int' in declaration of 'a' and deduced as 'std::initializer_list' in declaration of 'b'}} - auto (*fp)(int) -> int, i = 0; // expected-error {{declaration with trailing return type must be the only declaration in its group}} + auto a = 5, b = {1, 2}; + // since-cxx11-error@-1 {{'auto' deduced as 'int' in declaration of 'a' and deduced as 'std::initializer_list' in declaration of 'b'}} + auto (*fp)(int) -> int, i = 0; + // since-cxx11-error@-1 {{declaration with trailing return type must be the only declaration in its group}} #endif } -namespace dr1358 { // dr1358: yes +namespace dr1358 { // dr1358: 3.1 #if __cplusplus >= 201103L struct Lit { constexpr operator int() const { return 0; } }; - struct NonLit { NonLit(); operator int(); }; // expected-note 2{{no constexpr constructors}} + struct NonLit { NonLit(); operator int(); }; // #dr1358-NonLit struct NonConstexprConv { constexpr operator int() const; }; struct Virt { virtual int f(int) const; }; @@ -336,8 +409,12 @@ namespace dr1358 { // dr1358: yes // But the corresponding non-template cases are rejected. struct B : Virt { int member; - constexpr B(NonLit u) : member(u) {} // expected-error {{not a literal type}} - constexpr NonLit f(NonLit u) const { return NonLit(); } // expected-error {{not a literal type}} + constexpr B(NonLit u) : member(u) {} + // since-cxx11-error@-1 {{constexpr constructor's 1st parameter type 'NonLit' is not a literal type}} + // since-cxx11-note@#dr1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} + constexpr NonLit f(NonLit u) const { return NonLit(); } + // since-cxx11-error@-1 {{constexpr function's return type 'NonLit' is not a literal type}} + // since-cxx11-note@#dr1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} }; #endif } @@ -345,65 +422,113 @@ namespace dr1358 { // dr1358: yes namespace dr1359 { // dr1359: 3.5 #if __cplusplus >= 201103L union A { constexpr A() = default; }; - union B { constexpr B() = default; int a; }; // expected-error {{not constexpr}} expected-note 2{{candidate}} - union C { constexpr C() = default; int a, b; }; // expected-error {{not constexpr}} expected-note 2{{candidate}} - struct X { constexpr X() = default; union {}; }; // expected-error {{does not declare anything}} - struct Y { constexpr Y() = default; union { int a; }; }; // expected-error {{not constexpr}} expected-note 2{{candidate}} + union B { constexpr B() = default; int a; }; // #dr1359-B + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} + union C { constexpr C() = default; int a, b; }; // #dr1359-C + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} + struct X { constexpr X() = default; union {}; }; + // since-cxx11-error@-1 {{declaration does not declare anything}} + struct Y { constexpr Y() = default; union { int a; }; }; // #dr1359-Y + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} constexpr A a = A(); - constexpr B b = B(); // expected-error {{no matching}} - constexpr C c = C(); // expected-error {{no matching}} + constexpr B b = B(); + // cxx11-17-error@-1 {{no matching constructor for initialization of 'B'}} + // cxx11-17-note@#dr1359-B {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}} + // cxx11-17-note@#dr1359-B {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}} + constexpr C c = C(); + // cxx11-17-error@-1 {{no matching constructor for initialization of 'C'}} + // cxx11-17-note@#dr1359-C {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}} + // cxx11-17-note@#dr1359-C {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}} constexpr X x = X(); - constexpr Y y = Y(); // expected-error {{no matching}} + constexpr Y y = Y(); + // cxx11-17-error@-1 {{no matching constructor for initialization of 'Y'}} + // cxx11-17-note@#dr1359-Y {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}} + // cxx11-17-note@#dr1359-Y {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}} #endif } namespace dr1388 { // dr1388: 4 - template void f(T..., A); // expected-note 1+{{candidate}} expected-error 0-1{{C++11}} - template void g(T..., int); // expected-note 1+{{candidate}} expected-error 0-1{{C++11}} - template void h(T..., A); // expected-note 1+{{candidate}} expected-error 0-1{{C++11}} + template void f(T..., A); // #dr1388-f + // cxx98-error@-1 {{variadic templates are a C++11 extension}} + template void g(T..., int); // #dr1388-g + // cxx98-error@-1 {{variadic templates are a C++11 extension}} + template void h(T..., A); // #dr1388-h + // cxx98-error@-1 {{variadic templates are a C++11 extension}} void test_f() { f(0); // ok, trailing parameter pack deduced to empty - f(0, 0); // expected-error {{no matching}} + f(0, 0); + // expected-error@-1 {{no matching function for call to 'f'}} + // expected-note@#dr1388-f {{candidate function [with A = int, T = <>] not viable: requires 1 argument, but 2 were provided}} f(0); - f(0, 0); // expected-error {{no matching}} + f(0, 0); + // expected-error@-1 {{no matching function for call to 'f'}} + // expected-note@#dr1388-f {{candidate function [with A = int, T = <>] not viable: requires 1 argument, but 2 were provided}} f(0, 0); - f(0, 0); // expected-error {{no matching}} + f(0, 0); + // expected-error@-1 {{no matching function for call to 'f'}} + // expected-note@#dr1388-f {{candidate function [with A = int, T = ] not viable: requires 3 arguments, but 2 were provided}} g(0); - g(0, 0); // expected-error {{no matching}} + g(0, 0); + // expected-error@-1 {{no matching function for call to 'g'}} + // expected-note@#dr1388-g {{candidate function [with T = <>] not viable: requires 1 argument, but 2 were provided}} g<>(0); - g(0); // expected-error {{no matching}} + g(0); + // expected-error@-1 {{no matching function for call to 'g'}} + // expected-note@#dr1388-g {{candidate function [with T = ] not viable: requires 2 arguments, but 1 was provided}} g(0, 0); h(0); - h(0, 0); // expected-error {{no matching}} + h(0, 0); + // expected-error@-1 {{no matching function for call to 'h'}} + // expected-note@#dr1388-h {{candidate function [with T = <>, A = int] not viable: requires 1 argument, but 2 were provided}} h(0, 0); - h(0, 0); // expected-error {{no matching}} + h(0, 0); + // expected-error@-1 {{no matching function for call to 'h'}} + // expected-note@#dr1388-h {{candidate template ignored: couldn't infer template argument 'A'}} } // A non-trailing parameter pack is still a non-deduced context, even though // we know exactly how many arguments correspond to it. template struct pair {}; - template struct tuple { typedef char type; }; // expected-error 0-2{{C++11}} - template void f_pair_1(pair..., int); // expected-error 0-2{{C++11}} expected-note {{[with T = ]: deduced incomplete pack <(no value), (no value)> for template parameter 'U'}} - template void f_pair_2(pair..., U); // expected-error 0-2{{C++11}} - template void f_pair_3(pair..., tuple); // expected-error 0-2{{C++11}} expected-note {{deduced packs of different lengths for parameter 'U' (<(no value), (no value)> vs. )}} - template void f_pair_4(pair..., T...); // expected-error 0-2{{C++11}} expected-note {{ vs. }} + template struct tuple { typedef char type; }; // + // cxx98-error@-1 {{variadic templates are a C++11 extension}} + template void f_pair_1(pair..., int); // #dr1388-f-1 + // cxx98-error@-1 {{variadic templates are a C++11 extension}} + // cxx98-error@-2 {{variadic templates are a C++11 extension}} + template void f_pair_2(pair..., U); + // cxx98-error@-1 {{variadic templates are a C++11 extension}} + template void f_pair_3(pair..., tuple); // #dr1388-f-3 + // cxx98-error@-1 {{variadic templates are a C++11 extension}} + // cxx98-error@-2 {{variadic templates are a C++11 extension}} + template void f_pair_4(pair..., T...); // #dr1388-f-4 + // cxx98-error@-1 {{variadic templates are a C++11 extension}} void g(pair a, pair b, tuple c) { - f_pair_1(a, b, 0); // expected-error {{no match}} + f_pair_1(a, b, 0); + // expected-error@-1 {{no matching function for call to 'f_pair_1'}} + // expected-note@#dr1388-f-1 {{candidate template ignored: substitution failure [with T = ]: deduced incomplete pack <(no value), (no value)> for template parameter 'U'}} f_pair_2(a, b, 0); f_pair_3(a, b, c); - f_pair_3(a, b, tuple()); // expected-error {{no match}} + f_pair_3(a, b, tuple()); + // expected-error@-1 {{no matching function for call to 'f_pair_3'}} + // expected-note@#dr1388-f-3 {{candidate template ignored: deduced packs of different lengths for parameter 'U' (<(no value), (no value)> vs. )}} f_pair_4(a, b, 0, 0L); - f_pair_4(a, b, 0, 0L, "foo"); // expected-error {{no match}} + f_pair_4(a, b, 0, 0L, "foo"); + // expected-error@-1 {{no matching function for call to 'f_pair_4'}} + // expected-note@#dr1388-f-4 {{candidate template ignored: deduced packs of different lengths for parameter 'T' ( vs. )}} } } namespace dr1391 { // dr1391: partial struct A {}; struct B : A {}; - template struct C { C(int); typename T::error error; }; // expected-error 2{{'::'}} + template struct C { C(int); typename T::error error; }; // #dr1391-C + // expected-error@#dr1391-C {{type 'int' cannot be used prior to '::' because it has no members}} + // expected-note@#dr1391-b {{in instantiation of template class 'dr1391::C' requested here}} + // expected-note@#dr1391-b {{while substituting deduced template arguments into function template 'b' [with T = int]}} + // expected-error@#dr1391-C {{type 'double' cannot be used prior to '::' because it has no members}} + // expected-note@#dr1391-c {{in instantiation of template class 'dr1391::C' requested here}} template struct D {}; // No deduction is performed for parameters with no deducible template-parameters, therefore types do not need to match. @@ -444,28 +569,36 @@ namespace dr1391 { // dr1391: partial void test_b() { b(0, 0); // ok, deduction fails prior to forming a conversion sequence and instantiating C // FIXME: The "while substituting" note should point at the overload candidate. - b(0, 0); // expected-note {{instantiation of}} expected-note {{while substituting}} + b(0, 0); // #dr1391-b } template struct Id { typedef T type; }; template void c(T, typename Id >::type); void test_c() { // Implicit conversion sequences for dependent types are checked later. - c(0.0, 0); // expected-note {{instantiation of}} + c(0.0, 0); // #dr1391-c } namespace partial_ordering { // FIXME: Second template should be considered more specialized because non-dependent parameter is ignored. - template int a(T, short) = delete; // expected-error 0-1{{extension}} expected-note {{candidate}} - template int a(T*, char); // expected-note {{candidate}} - int test_a = a((int*)0, 0); // FIXME: expected-error {{ambiguous}} + template int a(T, short) = delete; // #dr1391-a-short + // cxx98-error@-1 {{deleted function definitions are a C++11 extension}} + template int a(T*, char); // #dr1391-a-char + int test_a = a((int*)0, 0); + // expected-error@-1 {{call to 'a' is ambiguous}} FIXME + // expected-note@#dr1391-a-short {{candidate function [with T = int *] has been explicitly deleted}} + // expected-note@#dr1391-a-char {{candidate function [with T = int]}} // FIXME: Second template should be considered more specialized: // deducing #1 from #2 ignores the second P/A pair, so deduction succeeds, // deducing #2 from #1 fails to deduce T, so deduction fails. - template int b(T, int) = delete; // expected-error 0-1{{extension}} expected-note {{candidate}} - template int b(T*, U); // expected-note {{candidate}} - int test_b = b((int*)0, 0); // FIXME: expected-error {{ambiguous}} + template int b(T, int) = delete; // #dr1391-b-int + // cxx98-error@-1 {{deleted function definitions are a C++11 extension}} + template int b(T*, U); // #dr1391-b-U + int test_b = b((int*)0, 0); + // expected-error@-1 {{call to 'b' is ambiguous}} FIXME + // expected-note@#dr1391-b-int {{candidate function [with T = int *] has been explicitly deleted}} + // expected-note@#dr1391-b-U {{candidate function [with T = int, U = int]}} // Unintended consequences: because partial ordering does not consider // explicit template arguments, and deduction from a non-dependent type @@ -499,26 +632,26 @@ namespace dr1395 { // dr1395: 16 namespace dr1397 { // dr1397: 3.2 #if __cplusplus >= 201103L -struct A { // #dr1397-struct-A - void *p = A{}; // #dr1397-void-p -#if __cplusplus == 201103L - // expected-error@#dr1397-struct-A {{default member initializer for 'p' needed within definition of enclosing class 'A' outside of member functions}} - // expected-note@#dr1397-void-p {{in evaluation of exception specification for 'dr1397::A::A' needed here}} - // expected-note@#dr1397-void-p {{default member initializer declared here}} -#elif __cplusplus >= 201402L - // expected-error@#dr1397-void-p {{default member initializer for 'p' needed within definition of enclosing class 'A' outside of member functions}} - // expected-note@#dr1397-void-p {{default member initializer declared here}} -#endif +struct A { +// cxx11-error@-1 {{default member initializer for 'p' needed within definition of enclosing class 'A' outside of member functions}} +// cxx11-note@#dr1397-p {{in evaluation of exception specification for 'dr1397::A::A' needed here}} +// cxx11-note@#dr1397-p {{default member initializer declared here}} + void *p = A{}; // #dr1397-p + // since-cxx14-error@-1 {{default member initializer for 'p' needed within definition of enclosing class 'A' outside of member functions}} + // since-cxx14-note@-2 {{default member initializer declared here}} operator void*() const { return nullptr; } }; #endif } // namespace dr1397 namespace dr1399 { // dr1399: dup 1388 - template void f(T..., int, T...) {} // expected-note {{candidate}} expected-error 0-1{{C++11}} + template void f(T..., int, T...) {} // #dr1399-f + // cxx98-error@-1 {{variadic templates are a C++11 extension}} void g() { f(0); f(0, 0, 0); - f(0, 0, 0); // expected-error {{no match}} + f(0, 0, 0); + // expected-error@-1 {{no matching function for call to 'f'}} + // expected-note@#dr1399-f {{candidate template ignored: deduced packs of different lengths for parameter 'T' (<> vs. )}} } } diff --git a/clang/test/CXX/drs/dr14xx.cpp b/clang/test/CXX/drs/dr14xx.cpp index ea41a03d3587a80284b7557f6e403e055bcfbbcf..d262f6f9dcab796f0d7105d08496e677a8a2c5e3 100644 --- a/clang/test/CXX/drs/dr14xx.cpp +++ b/clang/test/CXX/drs/dr14xx.cpp @@ -1,8 +1,10 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2a %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx11-17,since-cxx11, -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx14-17,cxx11-17,since-cxx11,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,cxx14-17,cxx11-17,since-cxx11,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx11,since-cxx14,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11,since-cxx14,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx11,since-cxx14,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors namespace dr1413 { // dr1413: 12 template struct Check { @@ -15,24 +17,41 @@ namespace dr1413 { // dr1413: 12 void d(); void f() { - Check::type *var1; // expected-error {{undeclared identifier 'var1'}} - Check::type *var2; // ok, variable declaration expected-note 0+{{here}} - Check::type *var3; // expected-error {{undeclared identifier 'var3'}} - Check::type *var4; // expected-error {{undeclared identifier 'var4'}} + Check::type *var1; + // expected-error@-1 {{use of undeclared identifier 'var1'}} + + // ok, variable declaration + Check::type *var2; // #dr1413-var2 + Check::type *var3; + // expected-error@-1 {{use of undeclared identifier 'var3'}} + // expected-note@#dr1413-var2 {{'var2' declared here}} + Check::type *var4; + // expected-error@-1 {{use of undeclared identifier 'var4'}} + // expected-note@#dr1413-var2 {{'var2' declared here}} + // value-dependent because of the implied type-dependent 'this->', not because of 'd' - Check::type *var5; // expected-error {{undeclared identifier 'var5'}} + Check::type *var5; + // expected-error@-1 {{use of undeclared identifier 'var5'}} + // expected-note@#dr1413-var2 {{'var2' declared here}} + // value-dependent because of the value-dependent '&' operator, not because of 'A::d' - Check::type *var5; // expected-error {{undeclared identifier 'var5'}} + Check::type *var5; + // expected-error@-1 {{use of undeclared identifier 'var5'}} + // expected-note@#dr1413-var2 {{'var2' declared here}} } }; } namespace dr1423 { // dr1423: 11 #if __cplusplus >= 201103L - bool b1 = nullptr; // expected-error {{cannot initialize}} - bool b2(nullptr); // expected-warning {{implicit conversion of nullptr constant to 'bool'}} - bool b3 = {nullptr}; // expected-error {{cannot initialize}} - bool b4{nullptr}; // expected-warning {{implicit conversion of nullptr constant to 'bool'}} + bool b1 = nullptr; + // since-cxx11-error@-1 {{cannot initialize a variable of type 'bool' with an rvalue of type 'std::nullptr_t'}} + bool b2(nullptr); + // since-cxx11-warning@-1 {{implicit conversion of nullptr constant to 'bool'}} + bool b3 = {nullptr}; + // since-cxx11-error@-1 {{cannot initialize a variable of type 'bool' with an rvalue of type 'std::nullptr_t'}} + bool b4{nullptr}; + // since-cxx11-warning@-1 {{implicit conversion of nullptr constant to 'bool'}} #endif } @@ -62,7 +81,8 @@ namespace dr1432 { // dr1432: 16 namespace dr1443 { // dr1443: yes struct A { int i; - A() { void foo(int=i); } // expected-error {{default argument references 'this'}} + A() { void foo(int=i); } + // expected-error@-1 {{default argument references 'this'}} }; } @@ -70,49 +90,54 @@ namespace dr1460 { // dr1460: 3.5 #if __cplusplus >= 201103L namespace DRExample { union A { - union {}; // expected-error {{does not declare anything}} - union {}; // expected-error {{does not declare anything}} + union {}; + // expected-error@-1 {{declaration does not declare anything}} + union {}; + // expected-error@-1 {{declaration does not declare anything}} constexpr A() {} }; constexpr A a = A(); union B { - union {}; // expected-error {{does not declare anything}} - union {}; // expected-error {{does not declare anything}} + union {}; + // expected-error@-1 {{declaration does not declare anything}} + union {}; + // expected-error@-1 {{declaration does not declare anything}} constexpr B() = default; }; constexpr B b = B(); union C { - union {}; // expected-error {{does not declare anything}} - union {}; // expected-error {{does not declare anything}} + union {}; + // expected-error@-1 {{declaration does not declare anything}} + union {}; + // expected-error@-1 {{declaration does not declare anything}} }; constexpr C c = C(); -#if __cplusplus > 201103L +#if __cplusplus >= 201403L constexpr void f() { C c; } static_assert((f(), true), ""); #endif } union A {}; - union B { int n; }; // expected-note 0+{{here}} + union B { int n; }; // #dr1460-B union C { int n = 0; }; - struct D { union {}; }; // expected-error {{does not declare anything}} - struct E { union { int n; }; }; // expected-note 0+{{here}} + struct D { union {}; }; + // expected-error@-1 {{declaration does not declare anything}} + struct E { union { int n; }; }; // #dr1460-E struct F { union { int n = 0; }; }; struct X { friend constexpr A::A() noexcept; friend constexpr B::B() noexcept; -#if __cplusplus <= 201703L - // expected-error@-2 {{follows non-constexpr declaration}} -#endif + // cxx11-17-error@-1 {{constexpr declaration of 'B' follows non-constexpr declaration}} + // cxx11-17-note@#dr1460-B {{previous declaration is here}} friend constexpr C::C() noexcept; friend constexpr D::D() noexcept; friend constexpr E::E() noexcept; -#if __cplusplus <= 201703L - // expected-error@-2 {{follows non-constexpr declaration}} -#endif + // cxx11-17-error@-1 {{constexpr declaration of 'E' follows non-constexpr declaration}} + // cxx11-17-note@#dr1460-E {{previous declaration is here}} friend constexpr F::F() noexcept; }; @@ -128,79 +153,77 @@ namespace dr1460 { // dr1460: 3.5 namespace Defaulted { union A { constexpr A() = default; }; union B { int n; constexpr B() = default; }; -#if __cplusplus <= 201703L - // expected-error@-2 {{not constexpr}} -#endif + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} union C { int n = 0; constexpr C() = default; }; - struct D { union {}; constexpr D() = default; }; // expected-error {{does not declare anything}} + struct D { union {}; constexpr D() = default; }; + // expected-error@-1 {{declaration does not declare anything}} struct E { union { int n; }; constexpr E() = default; }; -#if __cplusplus <= 201703L - // expected-error@-2 {{not constexpr}} -#endif + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} struct F { union { int n = 0; }; constexpr F() = default; }; struct G { union { int n = 0; }; union { int m; }; constexpr G() = default; }; -#if __cplusplus <= 201703L - // expected-error@-2 {{not constexpr}} -#endif + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} struct H { union { int n = 0; }; - union { // expected-note 0-2{{member not initialized}} + union { // #dr1460-H-union int m; }; constexpr H() {} -#if __cplusplus <= 201703L - // expected-error@-2 {{initialize all members}} -#endif + // cxx11-17-error@-1 {{constexpr constructor that does not initialize all members is a C++20 extension}} + // cxx11-17-note@#dr1460-H-union {{member not initialized by constructor}} constexpr H(bool) : m(1) {} constexpr H(char) : n(1) {} -#if __cplusplus <= 201703L - // expected-error@-2 {{initialize all members}} -#endif + // cxx11-17-error@-1 {{constexpr constructor that does not initialize all members is a C++20 extension}} + // cxx11-17-note@#dr1460-H-union {{member not initialized by constructor}} constexpr H(double) : m(1), n(1) {} }; } -#if __cplusplus > 201103L +#if __cplusplus >= 201403L template constexpr bool check() { - T t; -#if __cplusplus <= 201703L - // expected-note-re@-2 2{{non-constexpr constructor '{{[BE]}}'}} -#endif + T t; // #dr1460-t return true; } static_assert(check(), ""); - static_assert(check(), ""); -#if __cplusplus <= 201703L - // expected-error@-2 {{constant}} expected-note@-2 {{in call}} -#endif + static_assert(check(), ""); // #dr1460-check-B + // cxx14-17-error@-1 {{static assertion expression is not an integral constant expression}} + // cxx14-17-note@#dr1460-t {{non-constexpr constructor 'B' cannot be used in a constant expression}} + // cxx14-17-note@#dr1460-check-B {{in call to 'check()'}} + // cxx14-17-note@#dr1460-B {{declared here}} static_assert(check(), ""); static_assert(check(), ""); - static_assert(check(), ""); -#if __cplusplus <= 201703L - // expected-error@-2 {{constant}} expected-note@-2 {{in call}} -#endif + static_assert(check(), ""); // #dr1460-check-E + // cxx14-17-error@-1 {{static assertion expression is not an integral constant expression}} + // cxx14-17-note@#dr1460-t {{non-constexpr constructor 'E' cannot be used in a constant expression}} + // cxx14-17-note@#dr1460-check-E {{in call to 'check()'}} + // cxx14-17-note@#dr1460-E {{declared here}} static_assert(check(), ""); #endif union G { - int a = 0; // expected-note {{previous initialization is here}} - int b = 0; // expected-error {{initializing multiple members of union}} + int a = 0; // #dr1460-G-a + int b = 0; + // expected-error@-1 {{initializing multiple members of union}} + // expected-note@#dr1460-G-a {{previous initialization is here}} }; union H { union { - int a = 0; // expected-note {{previous initialization is here}} + int a = 0; // #dr1460-H-a }; union { - int b = 0; // expected-error {{initializing multiple members of union}} + int b = 0; + // expected-error@-1 {{initializing multiple members of union}} + // expected-note@#dr1460-H-a {{previous initialization is here}} }; }; struct I { union { - int a = 0; // expected-note {{previous initialization is here}} - int b = 0; // expected-error {{initializing multiple members of union}} + int a = 0; // #dr1460-I-a + int b = 0; + // expected-error@-1 {{initializing multiple members of union}} + // expected-note@#dr1460-I-a {{previous initialization is here}} }; }; struct J { @@ -223,14 +246,24 @@ namespace dr1460 { // dr1460: 3.5 constexpr B(const char*) {} }; static_assert(B().a == 1, ""); - static_assert(B().b == 2, ""); // expected-error {{constant}} expected-note {{read of}} - static_assert(B('x').a == 0, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(B().b == 2, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'b' of union with active member 'a' is not allowed in a constant expression}} + static_assert(B('x').a == 0, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'a' of union with active member 'b' is not allowed in a constant expression}} static_assert(B('x').b == 4, ""); - static_assert(B(123).b == 2, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(B(123).b == 2, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'b' of union with active member 'c' is not allowed in a constant expression}} static_assert(B(123).c == 3, ""); - static_assert(B("").a == 1, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(B("").a == 1, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'a' of union with active member 'b' is not allowed in a constant expression}} static_assert(B("").b == 2, ""); - static_assert(B("").c == 3, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(B("").c == 3, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'c' of union with active member 'b' is not allowed in a constant expression}} struct C { union { int a, b = 2, c; }; @@ -243,31 +276,55 @@ namespace dr1460 { // dr1460: 3.5 }; static_assert(C().a == 1, ""); - static_assert(C().b == 2, ""); // expected-error {{constant}} expected-note {{read of}} - static_assert(C().d == 4, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C().b == 2, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'b' of union with active member 'a' is not allowed in a constant expression}} + static_assert(C().d == 4, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'd' of union with active member 'e' is not allowed in a constant expression}} static_assert(C().e == 5, ""); - static_assert(C('x').b == 2, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C('x').b == 2, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'b' of union with active member 'c' is not allowed in a constant expression}} static_assert(C('x').c == 3, ""); - static_assert(C('x').d == 4, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C('x').d == 4, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'd' of union with active member 'e' is not allowed in a constant expression}} static_assert(C('x').e == 5, ""); static_assert(C(1).b == 2, ""); - static_assert(C(1).c == 3, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C(1).c == 3, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'c' of union with active member 'b' is not allowed in a constant expression}} static_assert(C(1).d == 4, ""); - static_assert(C(1).e == 5, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C(1).e == 5, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'e' of union with active member 'd' is not allowed in a constant expression}} static_assert(C(1.f).b == 2, ""); - static_assert(C(1.f).c == 3, ""); // expected-error {{constant}} expected-note {{read of}} - static_assert(C(1.f).e == 5, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C(1.f).c == 3, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'c' of union with active member 'b' is not allowed in a constant expression}} + static_assert(C(1.f).e == 5, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'e' of union with active member 'f' is not allowed in a constant expression}} static_assert(C(1.f).f == 6, ""); - static_assert(C("").a == 1, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C("").a == 1, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'a' of union with active member 'b' is not allowed in a constant expression}} static_assert(C("").b == 2, ""); - static_assert(C("").c == 3, ""); // expected-error {{constant}} expected-note {{read of}} - static_assert(C("").d == 4, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C("").c == 3, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'c' of union with active member 'b' is not allowed in a constant expression}} + static_assert(C("").d == 4, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'd' of union with active member 'e' is not allowed in a constant expression}} static_assert(C("").e == 5, ""); - static_assert(C("").f == 6, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C("").f == 6, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'f' of union with active member 'e' is not allowed in a constant expression}} struct D; extern const D d; @@ -315,8 +372,10 @@ namespace std { const _E* end() const {return __begin_ + __size_;} }; } // std +#endif namespace dr1467 { // dr1467: 3.7 c++11 +#if __cplusplus >= 201103L // Note that the change to [over.best.ics] was partially undone by DR2076; // the resulting rule is tested with the tests for that change. @@ -382,9 +441,12 @@ namespace dr1467 { // dr1467: 3.7 c++11 X x; X x2{x}; - void f1(int); // expected-note {{candidate function}} - void f1(std::initializer_list) = delete; // expected-note {{candidate function has been explicitly deleted}} - void g1() { f1({42}); } // expected-error {{call to deleted function 'f1'}} + void f1(int); // #dr1467-f1 + void f1(std::initializer_list) = delete; // #dr1467-f1-deleted + void g1() { f1({42}); } + // since-cxx11-error@-1 {{call to deleted function 'f1'}} + // since-cxx11-note@#dr1467-f1 {{candidate function}} + // since-cxx11-note@#dr1467-f1-deleted {{candidate function has been explicitly deleted}} template struct Pair { @@ -394,9 +456,12 @@ namespace dr1467 { // dr1467: 3.7 c++11 String(const char *); }; - void f2(Pair); // expected-note {{candidate function}} - void f2(std::initializer_list) = delete; // expected-note {{candidate function has been explicitly deleted}} - void g2() { f2({"foo", "bar"}); } // expected-error {{call to deleted function 'f2'}} + void f2(Pair); // #dr1467-f2 + void f2(std::initializer_list) = delete; // #dr1467-f2-deleted + void g2() { f2({"foo", "bar"}); } + // since-cxx11-error@-1 {{call to deleted function 'f2'}} + // since-cxx11-note@#dr1467-f2 {{candidate function}} + // since-cxx11-note@#dr1467-f2-deleted {{candidate function has been explicitly deleted}} } // dr_example namespace nonaggregate { @@ -453,84 +518,151 @@ namespace dr1467 { // dr1467: 3.7 c++11 } } // namespace NonAmbiguous -#if __cplusplus >= 201103L namespace StringLiterals { // When the array size is 4 the call will attempt to bind an lvalue to an // rvalue and fail. Therefore #2 will be called. (rsmith will bring this // issue to CWG) - void f(const char(&&)[4]); // expected-note 2 {{expects an rvalue}} expected-note 3 {{no known conversion}} - void f(const char(&&)[5]) = delete; // expected-note 2 {{candidate function has been explicitly deleted}} expected-note 3 {{no known conversion}} - void f(const wchar_t(&&)[4]); // expected-note {{expects an rvalue}} expected-note 4 {{no known conversion}} - void f(const wchar_t(&&)[5]) = delete; // expected-note {{candidate function has been explicitly deleted}} expected-note 4 {{no known conversion}} + void f(const char(&&)[4]); // #dr1467-f-char-4 + void f(const char(&&)[5]) = delete; // #dr1467-f-char-5 + void f(const wchar_t(&&)[4]); // #dr1467-f-wchar-4 + void f(const wchar_t(&&)[5]) = delete; // #dr1467-f-wchar-5 #if __cplusplus >= 202002L - void f2(const char8_t(&&)[4]); // expected-note {{expects an rvalue}} - void f2(const char8_t(&&)[5]) = delete; // expected-note {{candidate function has been explicitly deleted}} + void f2(const char8_t(&&)[4]); // #dr1467-f2-char8-4 + void f2(const char8_t(&&)[5]) = delete; // #dr1467-f2-char8-5 #endif - void f(const char16_t(&&)[4]); // expected-note {{expects an rvalue}} expected-note 4 {{no known conversion}} - void f(const char16_t(&&)[5]) = delete; // expected-note {{candidate function has been explicitly deleted}} expected-note 4 {{no known conversion}} - void f(const char32_t(&&)[4]); // expected-note {{expects an rvalue}} expected-note 4 {{no known conversion}} - void f(const char32_t(&&)[5]) = delete; // expected-note {{candidate function has been explicitly deleted}} expected-note 4 {{no known conversion}} + void f(const char16_t(&&)[4]); // #dr1467-f-char16-4 + void f(const char16_t(&&)[5]) = delete; // #dr1467-f-char16-5 + void f(const char32_t(&&)[4]); // #dr1467-f-char32-4 + void f(const char32_t(&&)[5]) = delete; // #dr1467-f-char32-5 void g() { - f({"abc"}); // expected-error {{call to deleted function 'f'}} - f({((("abc")))}); // expected-error {{call to deleted function 'f'}} - f({L"abc"}); // expected-error {{call to deleted function 'f'}} + f({"abc"}); + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr1467-f-char-5 {{candidate function has been explicitly deleted}} + // since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: expects an rvalue for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}} + f({((("abc")))}); + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr1467-f-char-5 {{candidate function has been explicitly deleted}} + // since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: expects an rvalue for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}} + f({L"abc"}); + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr1467-f-wchar-5 {{candidate function has been explicitly deleted}} + // since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char' for 1st argument}} + // since-cxx11-note@#dr1467-f-char-5 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char' for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: expects an rvalue for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-5 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char32_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-5 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char32_t' for 1st argument}} #if __cplusplus >= 202002L - f2({u8"abc"}); // expected-error {{call to deleted function 'f2'}} + f2({u8"abc"}); + // since-cxx20-error@-1 {{call to deleted function 'f2'}} + // since-cxx20-note@#dr1467-f2-char8-5 {{candidate function has been explicitly deleted}} + // since-cxx20-note@#dr1467-f2-char8-4 {{candidate function not viable: expects an rvalue for 1st argument}} #endif - f({uR"(abc)"}); // expected-error {{call to deleted function 'f'}} - f({(UR"(abc)")}); // expected-error {{call to deleted function 'f'}} + f({uR"(abc)"}); + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr1467-f-char16-5 {{candidate function has been explicitly deleted}} + // since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char' for 1st argument}} + // since-cxx11-note@#dr1467-f-char-5 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char' for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: expects an rvalue for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char32_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-5 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char32_t' for 1st argument}} + f({(UR"(abc)")}); + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr1467-f-char32-5 {{candidate function has been explicitly deleted}} + // since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char' for 1st argument}} + // since-cxx11-note@#dr1467-f-char-5 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char' for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-5 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: expects an rvalue for 1st argument}} } } // namespace StringLiterals #endif } // dr1467 -namespace dr1479 { // dr1479: yes - int operator"" _a(const char*, std::size_t = 0); // expected-error {{literal operator cannot have a default argument}} +namespace dr1479 { // dr1479: 3.1 +#if __cplusplus >= 201103L + int operator"" _a(const char*, std::size_t = 0); + // since-cxx11-error@-1 {{literal operator cannot have a default argument}} +#endif } -namespace dr1482 { // dr1482: yes +namespace dr1482 { // dr1482: 3.0 // NB: sup 2516, test reused there #if __cplusplus >= 201103L template struct S { typedef char I; }; enum E2 : S::I { e }; -// expected-error@-1 {{use of undeclared identifier 'E2'}} +// since-cxx11-error@-1 {{use of undeclared identifier 'E2'}} #endif } // namespace dr1482 namespace dr1490 { // dr1490: 3.7 c++11 +#if __cplusplus >= 201103L // List-initialization from a string literal - char s[4]{"abc"}; // Ok - std::initializer_list{"abc"}; // expected-error {{expected unqualified-id}}} + char s[4]{"abc"}; // Ok + std::initializer_list{"abc"}; + // since-cxx11-error@-1 {{expected unqualified-id}}} +#endif } // dr1490 namespace dr1495 { // dr1495: 4 +#if __cplusplus >= 201103L // Deduction succeeds in both directions. - template struct A {}; // expected-note {{template is declared here}} - template struct A {}; // expected-error {{class template partial specialization is not more specialized}} + template struct A {}; // #dr1495-A + template struct A {}; + // since-cxx11-error@-1 {{class template partial specialization is not more specialized than the primary template}} + // since-cxx11-note@#dr1495-A {{template is declared here}} // Primary template is more specialized. - template struct B {}; // expected-note {{template is declared here}} - template struct B {}; // expected-error {{not more specialized}} + template struct B {}; // #dr1495-B + template struct B {}; + // since-cxx11-error@-1 {{class template partial specialization is not more specialized than the primary template}} + // since-cxx11-note@#dr1495-B {{template is declared here}} // Deduction fails in both directions. - template struct C {}; // expected-note {{template is declared here}} - template struct C<0, Ts...> {}; // expected-error {{not more specialized}} + template struct C {}; // #dr1495-C + template struct C<0, Ts...> {}; + // since-cxx11-error@-1 {{class template partial specialization is not more specialized than the primary template}} + // since-cxx11-note@#dr1495-C {{template is declared here}} #if __cplusplus >= 201402L // Deduction succeeds in both directions. - template int a; // expected-note {{template is declared here}} - template int a; // expected-error {{variable template partial specialization is not more specialized}} + template int a; // #dr1495-a + template int a; + // since-cxx14-error@-1 {{variable template partial specialization is not more specialized than the primary template}} + // since-cxx14-note@#dr1495-a {{template is declared here}} // Primary template is more specialized. - template int b; // expected-note {{template is declared here}} - template int b; // expected-error {{not more specialized}} + template int b; // #dr1495-b + template int b; + // since-cxx14-error@-1 {{variable template partial specialization is not more specialized than the primary template}} + // since-cxx14-note@#dr1495-b {{template is declared here}} // Deduction fails in both directions. - template int c; // expected-note {{template is declared here}} - template int c<0, Ts...>; // expected-error {{not more specialized}} + template int c; // #dr1495-c + template int c<0, Ts...>; + // since-cxx14-error@-1 {{variable template partial specialization is not more specialized than the primary template}} + // since-cxx14-note@#dr1495-c {{template is declared here}} +#endif #endif } @@ -544,5 +676,3 @@ struct A { static_assert(__is_trivial(A), ""); #endif } - -#endif diff --git a/clang/test/CXX/drs/dr15xx.cpp b/clang/test/CXX/drs/dr15xx.cpp index c588891a75d6a88f27461a5dfca5edeb7081001f..007b42c74affb140788e38d0ddae1bd09548ba21 100644 --- a/clang/test/CXX/drs/dr15xx.cpp +++ b/clang/test/CXX/drs/dr15xx.cpp @@ -1,13 +1,18 @@ // RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,cxx14_17 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,cxx11-14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,cxx11-14,cxx14-17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors namespace dr1512 { // dr1512: 4 void f(char *p) { - if (p > 0) {} // expected-error {{ordered comparison between pointer and zero}} + if (p > 0) {} + // expected-error@-1 {{ordered comparison between pointer and zero ('char *' and 'int')}} #if __cplusplus >= 201103L - if (p > nullptr) {} // expected-error {{invalid operands}} + if (p > nullptr) {} + // since-cxx11-error@-1 {{invalid operands to binary expression ('char *' and 'std::nullptr_t')}} #endif } bool g(int **x, const int **y) { @@ -28,10 +33,20 @@ namespace dr1512 { // dr1512: 4 template void composite_pointer_type_is_ord() { composite_pointer_type_is_base(); - typedef __typeof(val() < val()) cmp; // cxx17-warning 2 {{ordered comparison of function pointers}} - typedef __typeof(val() <= val()) cmp; // cxx17-warning 2 {{ordered comparison of function pointers}} - typedef __typeof(val() > val()) cmp; // cxx17-warning 2 {{ordered comparison of function pointers}} - typedef __typeof(val() >= val()) cmp; // cxx17-warning 2 {{ordered comparison of function pointers}} + typedef __typeof(val() < val()) cmp; // #dr1512-lt + // since-cxx17-warning@#dr1512-lt {{ordered comparison of function pointers ('int (*)() noexcept' and 'int (*)()')}} + // since-cxx17-note@#dr1512-noexcept-1st {{in instantiation of function template specialization 'dr1512::composite_pointer_type_is_ord' requested here}} + // since-cxx17-warning@#dr1512-lt {{ordered comparison of function pointers ('int (*)()' and 'int (*)() noexcept')}} + // since-cxx17-note@#dr1512-noexcept-2nd {{in instantiation of function template specialization 'dr1512::composite_pointer_type_is_ord' requested here}} + typedef __typeof(val() <= val()) cmp; + // since-cxx17-warning@-1 {{ordered comparison of function pointers ('int (*)() noexcept' and 'int (*)()')}} + // since-cxx17-warning@-2 {{ordered comparison of function pointers ('int (*)()' and 'int (*)() noexcept')}} + typedef __typeof(val() > val()) cmp; + // since-cxx17-warning@-1 {{ordered comparison of function pointers ('int (*)() noexcept' and 'int (*)()')}} + // since-cxx17-warning@-2 {{ordered comparison of function pointers ('int (*)()' and 'int (*)() noexcept')}} + typedef __typeof(val() >= val()) cmp; + // since-cxx17-warning@-1 {{ordered comparison of function pointers ('int (*)() noexcept' and 'int (*)()')}} + // since-cxx17-warning@-2 {{ordered comparison of function pointers ('int (*)()' and 'int (*)() noexcept')}} typedef bool cmp; } @@ -77,10 +92,11 @@ namespace dr1512 { // dr1512: 4 composite_pointer_type_is_unord(); no_composite_pointer_type(); no_composite_pointer_type(); + // since-cxx20-warning@-1 {{volatile-qualified return type 'volatile int' is deprecated}} -#if __cplusplus > 201402 - composite_pointer_type_is_ord(); // expected-note {{requested here}} - composite_pointer_type_is_ord(); // expected-note {{requested here}} +#if __cplusplus >= 201703L + composite_pointer_type_is_ord(); // #dr1512-noexcept-1st + composite_pointer_type_is_ord(); // #dr1512-noexcept-2nd composite_pointer_type_is_unord(); composite_pointer_type_is_unord(); // FIXME: This looks like a standard defect; these should probably all have type 'int (B::*)()'. @@ -113,24 +129,40 @@ namespace dr1512 { // dr1512: 4 } #if __cplusplus >= 201103L - template struct Wrap { operator T(); }; // expected-note 4{{converted to type 'std::nullptr_t'}} expected-note 4{{converted to type 'int *'}} + template struct Wrap { operator T(); }; // #dr1512-Wrap void test_overload() { using nullptr_t = decltype(nullptr); void(Wrap() == Wrap()); void(Wrap() != Wrap()); - void(Wrap() < Wrap()); // expected-error {{invalid operands}} - void(Wrap() > Wrap()); // expected-error {{invalid operands}} - void(Wrap() <= Wrap()); // expected-error {{invalid operands}} - void(Wrap() >= Wrap()); // expected-error {{invalid operands}} + void(Wrap() < Wrap()); + // since-cxx11-error@-1 {{invalid operands to binary expression ('Wrap' (aka 'Wrap') and 'Wrap' (aka 'Wrap'))}} + void(Wrap() > Wrap()); + // since-cxx11-error@-1 {{invalid operands to binary expression ('Wrap' (aka 'Wrap') and 'Wrap' (aka 'Wrap'))}} + void(Wrap() <= Wrap()); + // since-cxx11-error@-1 {{invalid operands to binary expression ('Wrap' (aka 'Wrap') and 'Wrap' (aka 'Wrap'))}} + void(Wrap() >= Wrap()); + // since-cxx11-error@-1 {{invalid operands to binary expression ('Wrap' (aka 'Wrap') and 'Wrap' (aka 'Wrap'))}} // Under dr1213, this is ill-formed: we select the builtin operator<(int*, int*) // but then only convert as far as 'nullptr_t', which we then can't convert to 'int*'. void(Wrap() == Wrap()); void(Wrap() != Wrap()); - void(Wrap() < Wrap()); // expected-error {{invalid operands to binary expression ('Wrap' (aka 'Wrap') and 'Wrap')}} - void(Wrap() > Wrap()); // expected-error {{invalid operands}} - void(Wrap() <= Wrap()); // expected-error {{invalid operands}} - void(Wrap() >= Wrap()); // expected-error {{invalid operands}} + void(Wrap() < Wrap()); + // since-cxx11-error@-1 {{invalid operands to binary expression ('Wrap' (aka 'Wrap') and 'Wrap')}} + // since-cxx11-note@#dr1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}} + // since-cxx11-note@#dr1512-Wrap {{second operand was implicitly converted to type 'int *'}} + void(Wrap() > Wrap()); + // since-cxx11-error@-1 {{invalid operands}} + // since-cxx11-note@#dr1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}} + // since-cxx11-note@#dr1512-Wrap{{second operand was implicitly converted to type 'int *'}} + void(Wrap() <= Wrap()); + // since-cxx11-error@-1 {{invalid operands}} + // since-cxx11-note@#dr1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}} + // since-cxx11-note@#dr1512-Wrap {{second operand was implicitly converted to type 'int *'}} + void(Wrap() >= Wrap()); + // since-cxx11-error@-1 {{invalid operands}} + // since-cxx11-note@#dr1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}} + // since-cxx11-note@#dr1512-Wrap {{second operand was implicitly converted to type 'int *'}} } #endif } @@ -138,8 +170,10 @@ namespace dr1512 { // dr1512: 4 namespace dr1514 { // dr1514: 11 #if __cplusplus >= 201103L struct S { - enum E : int {}; // expected-note {{previous}} - enum E : int {}; // expected-error {{redefinition}} + enum E : int {}; // #dr1514-E + enum E : int {}; + // since-cxx11-error@-1 {{redefinition of 'E'}} + // since-cxx11-note@#dr1514-E {{previous definition is here}} }; S::E se; // OK, complete type, not zero-width bitfield. @@ -149,91 +183,142 @@ namespace dr1514 { // dr1514: 11 namespace dr1518 { // dr1518: 4 #if __cplusplus >= 201103L -struct Z0 { // expected-note 0+ {{candidate}} - explicit Z0() = default; // expected-note 0+ {{here}} +struct Z0 { // #dr1518-Z0 + explicit Z0() = default; // #dr1518-Z0-ctor }; -struct Z { // expected-note 0+ {{candidate}} - explicit Z(); // expected-note 0+ {{here}} - explicit Z(int); // expected-note {{not a candidate}} - explicit Z(int, int); // expected-note 0+ {{here}} +struct Z { // #dr1518-Z + explicit Z(); // #dr1518-Z-ctor + explicit Z(int); // #dr1518-Z-int + explicit Z(int, int); // #dr1518-Z-int-int }; -template int Eat(T); // expected-note 0+ {{candidate}} +template int Eat(T); // #dr1518-Eat Z0 a; Z0 b{}; -Z0 c = {}; // expected-error {{explicit in copy-initialization}} -int i = Eat({}); // expected-error {{no matching function for call to 'Eat'}} - -Z c2 = {}; // expected-error {{explicit in copy-initialization}} -int i2 = Eat({}); // expected-error {{no matching function for call to 'Eat'}} -Z a1 = 1; // expected-error {{no viable conversion}} +Z0 c = {}; +// since-cxx11-error@-1 {{chosen constructor is explicit in copy-initialization}} +// since-cxx11-note@#dr1518-Z0-ctor {{explicit constructor declared here}} +int i = Eat({}); +// since-cxx11-error@-1 {{no matching function for call to 'Eat'}} +// since-cxx11-note@#dr1518-Eat {{candidate function template not viable: cannot convert initializer list argument to 'Z0'}} + +Z c2 = {}; +// since-cxx11-error@-1 {{chosen constructor is explicit in copy-initialization}} +// since-cxx11-note@#dr1518-Z-ctor {{explicit constructor declared here}} +int i2 = Eat({}); +// since-cxx11-error@-1 {{no matching function for call to 'Eat'}} +// since-cxx11-note@#dr1518-Eat {{candidate function template not viable: cannot convert initializer list argument to 'Z'}} +Z a1 = 1; +// since-cxx11-error@-1 {{no viable conversion from 'int' to 'Z'}} +// since-cxx11-note@#dr1518-Z {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'int' to 'const Z &' for 1st argument}} +// since-cxx11-note@#dr1518-Z {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'int' to 'Z &&' for 1st argument}} +// since-cxx11-note@#dr1518-Z-int {{explicit constructor is not a candidate}} Z a3 = Z(1); Z a2(1); Z *p = new Z(1); Z a4 = (Z)1; Z a5 = static_cast(1); -Z a6 = {4, 3}; // expected-error {{explicit in copy-initialization}} +Z a6 = {4, 3}; +// since-cxx11-error@-1 {{chosen constructor is explicit in copy-initialization}} +// since-cxx11-note@#dr1518-Z-int-int {{explicit constructor declared here}} -struct UserProvidedBaseCtor { // expected-note 0+ {{candidate}} +struct UserProvidedBaseCtor { // #dr1518-U UserProvidedBaseCtor() {} }; -struct DoesntInheritCtor : UserProvidedBaseCtor { // expected-note 0+ {{candidate}} +struct DoesntInheritCtor : UserProvidedBaseCtor { // #dr1518-D-U int x; }; DoesntInheritCtor I{{}, 42}; -#if __cplusplus <= 201402L -// expected-error@-2 {{no matching constructor}} -#endif - -struct BaseCtor { BaseCtor() = default; }; // expected-note 0+ {{candidate}} -struct InheritsCtor : BaseCtor { // expected-note 1+ {{candidate}} - using BaseCtor::BaseCtor; // expected-note 2 {{inherited here}} +// cxx11-14-error@-1 {{no matching constructor for initialization of 'DoesntInheritCtor'}} +// cxx11-14-note@#dr1518-D-U {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}} +// cxx11-14-note@#dr1518-D-U {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}} +// cxx11-14-note@#dr1518-D-U {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 2 were provided}} + +struct BaseCtor { BaseCtor() = default; }; // #dr1518-BC +struct InheritsCtor : BaseCtor { // #dr1518-I + using BaseCtor::BaseCtor; // #dr1518-I-using int x; }; -InheritsCtor II = {{}, 42}; // expected-error {{no matching constructor}} +InheritsCtor II = {{}, 42}; +// since-cxx11-error@-1 {{no matching constructor for initialization of 'InheritsCtor'}} +// since-cxx11-note@#dr1518-BC {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}} +// since-cxx11-note@#dr1518-I-using {{constructor from base class 'BaseCtor' inherited here}} +// since-cxx11-note@#dr1518-BC {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}} +// since-cxx11-note@#dr1518-I-using {{constructor from base class 'BaseCtor' inherited here}} +// since-cxx11-note@#dr1518-I {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}} +// since-cxx11-note@#dr1518-I {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}} +// since-cxx11-note@#dr1518-I {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 2 were provided}} namespace std_example { struct A { - explicit A() = default; // expected-note 2{{declared here}} + explicit A() = default; // #dr1518-A }; struct B : A { - explicit B() = default; // expected-note 2{{declared here}} + explicit B() = default; // #dr1518-B }; struct C { - explicit C(); // expected-note 2{{declared here}} + explicit C(); // #dr1518-C }; struct D : A { C c; - explicit D() = default; // expected-note 2{{declared here}} + explicit D() = default; // #dr1518-D }; template void f() { T t; // ok T u{}; // ok - T v = {}; // expected-error 4{{explicit}} + T v = {}; // #dr1518-v + // since-cxx11-error@#dr1518-v {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-f-A {{in instantiation of function template specialization 'dr1518::std_example::f' requested here}} + // since-cxx11-note@#dr1518-A {{explicit constructor declared here}} + // since-cxx11-error@#dr1518-v {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-f-B {{in instantiation of function template specialization 'dr1518::std_example::f' requested here}} + // since-cxx11-note@#dr1518-B {{explicit constructor declared here}} + // since-cxx11-error@#dr1518-v {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-f-C {{in instantiation of function template specialization 'dr1518::std_example::f' requested here}} + // since-cxx11-note@#dr1518-C {{explicit constructor declared here}} + // since-cxx11-error@#dr1518-v {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-f-D {{in instantiation of function template specialization 'dr1518::std_example::f' requested here}} + // since-cxx11-note@#dr1518-D {{explicit constructor declared here}} } template void g() { - void x(T t); // expected-note 4{{parameter}} - x({}); // expected-error 4{{explicit}} + void x(T t); // #dr1518-x + x({}); // #dr1518-x-call + // since-cxx11-error@#dr1518-x-call {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-g-A {{in instantiation of function template specialization 'dr1518::std_example::g' requested here}} + // since-cxx11-note@#dr1518-A {{explicit constructor declared here}} + // since-cxx11-note@#dr1518-x {{passing argument to parameter 't' here}} + // since-cxx11-error@#dr1518-x-call {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-g-B {{in instantiation of function template specialization 'dr1518::std_example::g' requested here}} + // since-cxx11-note@#dr1518-B {{explicit constructor declared here}} + // since-cxx11-note@#dr1518-x {{passing argument to parameter 't' here}} + // since-cxx11-error@#dr1518-x-call {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-g-C {{in instantiation of function template specialization 'dr1518::std_example::g' requested here}} + // since-cxx11-note@#dr1518-C {{explicit constructor declared here}} + // since-cxx11-note@#dr1518-x {{passing argument to parameter 't' here}} + // since-cxx11-error@#dr1518-x-call {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-g-D {{in instantiation of function template specialization 'dr1518::std_example::g' requested here}} + // since-cxx11-note@#dr1518-D {{explicit constructor declared here}} + // since-cxx11-note@#dr1518-x {{passing argument to parameter 't' here}} } void test() { - f(); // expected-note {{instantiation of}} - f(); // expected-note {{instantiation of}} - f(); // expected-note {{instantiation of}} - f(); // expected-note {{instantiation of}} - g(); // expected-note {{instantiation of}} - g(); // expected-note {{instantiation of}} - g(); // expected-note {{instantiation of}} - g(); // expected-note {{instantiation of}} + f(); // #dr1518-f-A + f(); // #dr1518-f-B + f(); // #dr1518-f-C + f(); // #dr1518-f-D + g(); // #dr1518-g-A + g(); // #dr1518-g-B + g(); // #dr1518-g-C + g(); // #dr1518-g-D } } -#endif // __cplusplus >= 201103L +#endif // __cplusplus >= 201103L } -namespace dr1550 { // dr1550: yes +namespace dr1550 { // dr1550: 3.4 int f(bool b, int n) { return (b ? (throw 0) : n) + (b ? n : (throw 0)); } @@ -242,13 +327,16 @@ namespace dr1550 { // dr1550: yes namespace dr1558 { // dr1558: 12 #if __cplusplus >= 201103L template using first_of = T; - template first_of f(int); // expected-note {{'int' cannot be used prior to '::'}} - template void f(...) = delete; // expected-note {{deleted}} + template first_of f(int); // #dr1558-f + template void f(...) = delete; // #dr1558-f-deleted struct X { typedef void type; }; void test() { f(0); - f(0); // expected-error {{deleted}} + f(0); + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr1558-f-deleted {{candidate function [with T = int] has been explicitly deleted}} + // since-cxx11-note@#dr1558-f {{candidate template ignored: substitution failure [with T = int]: type 'int' cannot be used prior to '::' because it has no members}} } #endif } @@ -283,17 +371,25 @@ namespace dr1573 { // dr1573: 3.9 struct C { C(); constexpr C(int) {} }; struct D : C { using C::C; }; constexpr D d = D(0); // ok - struct E : C { using C::C; A a; }; // expected-note {{non-literal type}} - constexpr E e = E(0); // expected-error {{non-literal type}} + struct E : C { using C::C; A a; }; // #dr1573-E + constexpr E e = E(0); + // since-cxx11-error@-1 {{constexpr variable cannot have non-literal type 'const E'}} + // since-cxx11-note@#dr1573-E {{'E' is not literal because it has data member 'a' of non-literal type 'A'}} + // FIXME: This diagnostic is pretty bad; we should explain that the problem // is that F::c would be initialized by a non-constexpr constructor. - struct F : C { using C::C; C c; }; // expected-note {{here}} - constexpr F f = F(0); // expected-error {{constant expression}} expected-note {{constructor inherited from base class 'C'}} + struct F : C { using C::C; C c; }; // #dr1573-F + constexpr F f = F(0); + // since-cxx11-error@-1 {{constexpr variable 'f' must be initialized by a constant expression}} + // since-cxx11-note@-2 {{constructor inherited from base class 'C' cannot be used in a constant expression; derived class cannot be implicitly initialized}} + // since-cxx11-note@#dr1573-F {{declared here}} // inherited constructor is effectively deleted if the user-written constructor would be struct G { G(int); }; - struct H : G { using G::G; G g; }; // expected-note {{constructor inherited by 'H' is implicitly deleted because field 'g' has no default constructor}} - H h(0); // expected-error {{constructor inherited by 'H' from base class 'G' is implicitly deleted}} + struct H : G { using G::G; G g; }; // #dr1573-H + H h(0); + // since-cxx11-error@-1 {{constructor inherited by 'H' from base class 'G' is implicitly deleted}} + // since-cxx11-note@#dr1573-H {{constructor inherited by 'H' is implicitly deleted because field 'g' has no default constructor}} #endif } @@ -336,13 +432,15 @@ namespace std { typedef basic_string string; } // std +#endif namespace dr1579 { // dr1579: 3.9 +#if __cplusplus >= 201103L template struct GenericMoveOnly { GenericMoveOnly(); - template GenericMoveOnly(const GenericMoveOnly &) = delete; // expected-note 5 {{marked deleted here}} - GenericMoveOnly(const int &) = delete; // expected-note 2 {{marked deleted here}} + template GenericMoveOnly(const GenericMoveOnly &) = delete; // #dr1579-deleted-U + GenericMoveOnly(const int &) = delete; // #dr1579-deleted-int template GenericMoveOnly(GenericMoveOnly &&); GenericMoveOnly(int &&); }; @@ -369,17 +467,29 @@ GenericMoveOnly DR1579_Ineligible(int &AnInt, extern GenericMoveOnly ExternMove; if (0) - return AnInt; // expected-error{{invokes a deleted function}} + return AnInt; + // since-cxx11-error@-1 {{conversion function from 'int' to 'GenericMoveOnly' invokes a deleted function}} + // since-cxx11-note@#dr1579-deleted-int {{'GenericMoveOnly' has been explicitly marked deleted here}} else if (0) - return GlobalMO; // expected-error{{invokes a deleted function}} + return GlobalMO; + // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly' to 'GenericMoveOnly' invokes a deleted function}} + // since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly' has been explicitly marked deleted here}} else if (0) - return StaticMove; // expected-error{{invokes a deleted function}} + return StaticMove; + // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly' to 'GenericMoveOnly' invokes a deleted function}} + // since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly' has been explicitly marked deleted here}} else if (0) - return ExternMove; // expected-error{{invokes a deleted function}} + return ExternMove; + // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly' to 'GenericMoveOnly' invokes a deleted function}} + // since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly' has been explicitly marked deleted here}} else if (0) - return AnInt; // expected-error{{invokes a deleted function}} + return AnInt; + // since-cxx11-error@-1 {{conversion function from 'int' to 'GenericMoveOnly' invokes a deleted function}} + // since-cxx11-note@#dr1579-deleted-int {{'GenericMoveOnly' has been explicitly marked deleted here}} else - return CharMO; // expected-error{{invokes a deleted function}} + return CharMO; + // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly' to 'GenericMoveOnly' invokes a deleted function}} + // since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly' has been explicitly marked deleted here}} } auto DR1579_lambda_valid = [](GenericMoveOnly mo) -> @@ -389,24 +499,34 @@ auto DR1579_lambda_valid = [](GenericMoveOnly mo) -> auto DR1579_lambda_invalid = []() -> GenericMoveOnly { static GenericMoveOnly mo; - return mo; // expected-error{{invokes a deleted function}} + return mo; + // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly' to 'GenericMoveOnly' invokes a deleted function}} + // since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly' has been explicitly marked deleted here}} }; +#endif } // end namespace dr1579 namespace dr1584 { +#if __cplusplus >= 201103L // Deducing function types from cv-qualified types - template void f(const T *); // expected-note {{candidate template ignored}} + template void f(const T *); // #dr1584-f template void g(T *, const T * = 0); - template void h(T *) { T::error; } // expected-error {{no members}} + template void h(T *) { T::error; } + // since-cxx11-error@-1 {{type 'void ()' cannot be used prior to '::' because it has no members}} + // since-cxx11-note@#dr1584-h {{in instantiation of function template specialization 'dr1584::h' requested here}} template void h(const T *); void i() { - f(&i); // expected-error {{no matching function}} + f(&i); + // since-cxx11-error@-1 {{no matching function for call to 'f'}} + // since-cxx11-note@#dr1584-f {{candidate template ignored: could not match 'const T *' against 'void (*)()'}} g(&i); - h(&i); // expected-note {{here}} + h(&i); // #dr1584-h } +#endif } namespace dr1589 { // dr1589: 3.7 c++11 +#if __cplusplus >= 201103L // Ambiguous ranking of list-initialization sequences void f0(long, int=0); // Would makes selection of #0 ambiguous @@ -425,25 +545,35 @@ namespace dr1589 { // dr1589: 3.7 c++11 void g2() { f2({"foo","bar"}); } // chooses #4 namespace with_error { - void f0(long); // #0 - void f0(std::initializer_list); // #00 expected-note {{candidate function}} - void f0(std::initializer_list, int = 0); // expected-note {{candidate function}} - void g0() { f0({1L}); } // expected-error{{call to 'f0' is ambiguous}} - - void f1(int); // #1 - void f1(std::initializer_list); // #2 expected-note {{candidate function}} - void f1(std::initializer_list, int = 0); // expected-note {{candidate function}} - void g1() { f1({42}); } // expected-error{{call to 'f1' is ambiguous}} - - void f2(std::pair); // #3 - void f2(std::initializer_list); // #4 expected-note {{candidate function}} - void f2(std::initializer_list, int = 0); // expected-note {{candidate function}} - void g2() { f2({"foo","bar"}); } // expected-error{{call to 'f2' is ambiguous}} + void f0(long); + void f0(std::initializer_list); // #dr1589-f0-ilist + void f0(std::initializer_list, int = 0); // #dr1589-f0-ilist-int + void g0() { f0({1L}); } + // since-cxx11-error@-1 {{call to 'f0' is ambiguous}} + // since-cxx11-note@#dr1589-f0-ilist {{candidate function}} + // since-cxx11-note@#dr1589-f0-ilist-int {{candidate function}} + + void f1(int); + void f1(std::initializer_list); // #dr1589-f1-ilist + void f1(std::initializer_list, int = 0); // #dr1589-f1-ilist-long + void g1() { f1({42}); } + // since-cxx11-error@-1 {{call to 'f1' is ambiguous}} + // since-cxx11-note@#dr1589-f1-ilist {{candidate function}} + // since-cxx11-note@#dr1589-f1-ilist-long {{candidate function}} + + void f2(std::pair); + void f2(std::initializer_list); // #dr1589-f2-ilist + void f2(std::initializer_list, int = 0); // #dr1589-f2-ilist-int + void g2() { f2({"foo","bar"}); } + // since-cxx11-error@-1 {{call to 'f2' is ambiguous}} + // since-cxx11-note@#dr1589-f2-ilist {{candidate function}} + // since-cxx11-note@#dr1589-f2-ilist-int {{candidate function}} } - +#endif } // dr1589 -namespace dr1591 { //dr1591. Deducing array bound and element type from initializer list +namespace dr1591 { //dr1591. Deducing array bound and element type from initializer list +#if __cplusplus >= 201103L template int h(T const(&)[N]); int X = h({1,2,3}); // T deduced to int, N deduced to 3 @@ -451,8 +581,10 @@ namespace dr1591 { //dr1591. Deducing array bound and element type from initial int Y = j({42}); // T deduced to int, array bound not considered struct Aggr { int i; int j; }; - template int k(Aggr const(&)[N]); //expected-note{{not viable}} - int Y0 = k({1,2,3}); //expected-error{{no matching function}} + template int k(Aggr const(&)[N]); // #dr1591-k + int Y0 = k({1,2,3}); + // since-cxx11-error@-1 {{no matching function for call to 'k'}} + // since-cxx11-note@#dr1591-k {{candidate function [with N = 3] not viable: no known conversion from 'int' to 'const Aggr' for 1st argument}} int Z = k({{1},{2},{3}}); // OK, N deduced to 3 template int m(int const(&)[M][N]); @@ -463,54 +595,64 @@ namespace dr1591 { //dr1591. Deducing array bound and element type from initial namespace check_multi_dim_arrays { - template int ***f(const T (&a)[N][M][O]); //expected-note{{deduced conflicting values}} - template int **f(const T (&a)[N][M]); //expected-note{{couldn't infer}} + template int ***f(const T (&a)[N][M][O]); // #dr1591-f-3 + template int **f(const T (&a)[N][M]); // #dr1591-f-2 - template int *f(const T (&a)[N]); //expected-note{{couldn't infer}} + template int *f(const T (&a)[N]); // #dr1591-f-1 int ***p3 = f({ { {1,2}, {3, 4} }, { {5,6}, {7, 8} }, { {9,10}, {11, 12} } }); - int ***p33 = f({ { {1,2}, {3, 4} }, { {5,6}, {7, 8} }, { {9,10}, {11, 12, 13} } }); //expected-error{{no matching}} + int ***p33 = f({ { {1,2}, {3, 4} }, { {5,6}, {7, 8} }, { {9,10}, {11, 12, 13} } }); + // since-cxx11-error@-1 {{no matching function for call to 'f'}} + // since-cxx11-note@#dr1591-f-2 {{candidate template ignored: couldn't infer template argument 'T'}} + // since-cxx11-note@#dr1591-f-1 {{candidate template ignored: couldn't infer template argument 'T'}} + // since-cxx11-note@#dr1591-f-3 {{candidate template ignored: deduced conflicting values for parameter 'O' (2 vs. 3)}} int **p2 = f({ {1,2,3}, {3, 4, 5} }); int **p22 = f({ {1,2}, {3, 4} }); int *p1 = f({1, 2, 3}); } namespace check_multi_dim_arrays_rref { - template int ***f(T (&&a)[N][M][O]); //expected-note{{deduced conflicting values}} - template int **f(T (&&a)[N][M]); //expected-note{{couldn't infer}} + template int ***g(T (&&a)[N][M][O]); // #dr1591-g-3 + template int **g(T (&&a)[N][M]); // #dr1591-g-2 - template int *f(T (&&a)[N]); //expected-note{{couldn't infer}} - int ***p3 = f({ { {1,2}, {3, 4} }, { {5,6}, {7, 8} }, { {9,10}, {11, 12} } }); - int ***p33 = f({ { {1,2}, {3, 4} }, { {5,6}, {7, 8} }, { {9,10}, {11, 12, 13} } }); //expected-error{{no matching}} - int **p2 = f({ {1,2,3}, {3, 4, 5} }); - int **p22 = f({ {1,2}, {3, 4} }); - int *p1 = f({1, 2, 3}); + template int *g(T (&&a)[N]); // #dr1591-g-1 + int ***p3 = g({ { {1,2}, {3, 4} }, { {5,6}, {7, 8} }, { {9,10}, {11, 12} } }); + int ***p33 = g({ { {1,2}, {3, 4} }, { {5,6}, {7, 8} }, { {9,10}, {11, 12, 13} } }); + // since-cxx11-error@-1 {{no matching function for call to 'g'}} + // since-cxx11-note@#dr1591-g-2 {{candidate template ignored: couldn't infer template argument 'T'}} + // since-cxx11-note@#dr1591-g-1 {{candidate template ignored: couldn't infer template argument 'T'}} + // since-cxx11-note@#dr1591-g-3 {{candidate template ignored: deduced conflicting values for parameter 'O' (2 vs. 3)}} + int **p2 = g({ {1,2,3}, {3, 4, 5} }); + int **p22 = g({ {1,2}, {3, 4} }); + int *p1 = g({1, 2, 3}); } namespace check_arrays_of_init_list { - template float *f(const std::initializer_list (&)[N]); - template double *f(const T(&)[N]); - double *p = f({1, 2, 3}); - float *fp = f({{1}, {1, 2}, {1, 2, 3}}); + template float *h(const std::initializer_list (&)[N]); + template double *h(const T(&)[N]); + double *p = h({1, 2, 3}); + float *fp = h({{1}, {1, 2}, {1, 2, 3}}); } namespace core_reflector_28543 { - template int *f(T (&&)[N]); // #1 - template char *f(std::initializer_list &&); //#2 - template int **f(T (&&)[N][M]); //#3 expected-note{{candidate}} - template char **f(std::initializer_list (&&)[N]); //#4 expected-note{{candidate}} + template int *i(T (&&)[N]); // #1 + template char *i(std::initializer_list &&); // #2 + template int **i(T (&&)[N][M]); // #3 #dr1591-i-2 + template char **i(std::initializer_list (&&)[N]); // #4 #dr1591-i-1 - template short *f(T (&&)[2]); //#5 + template short *i(T (&&)[2]); // #5 template using Arr = T[]; - char *pc = f({1, 2, 3}); // OK prefer #2 via 13.3.3.2 [over.ics.rank] - char *pc2 = f({1, 2}); // #2 also - int *pi = f(Arr{1, 2, 3}); // OK prefer #1 + char *pc = i({1, 2, 3}); // OK prefer #2 via 13.3.3.2 [over.ics.rank] + char *pc2 = i({1, 2}); // #2 also + int *pi = i(Arr{1, 2, 3}); // OK prefer #1 - void *pv1 = f({ {1, 2, 3}, {4, 5, 6} }); // expected-error{{ambiguous}} btw 3 & 4 - char **pcc = f({ {1}, {2, 3} }); // OK #4 + void *pv1 = i({ {1, 2, 3}, {4, 5, 6} }); // ambiguous btw 3 & 4 + // since-cxx11-error@-1 {{call to 'i' is ambiguous}} + // since-cxx11-note@#dr1591-i-2 {{candidate function [with T = int, N = 2, M = 3]}} + // since-cxx11-note@#dr1591-i-1 {{candidate function [with T = int, N = 2]}} + char **pcc = i({ {1}, {2, 3} }); // OK #4 - short *ps = f(Arr{1, 2}); // OK #5 + short *ps = i(Arr{1, 2}); // OK #5 } -} // dr1591 - #endif +} // dr1591 diff --git a/clang/test/CXX/drs/dr16xx.cpp b/clang/test/CXX/drs/dr16xx.cpp index 463353dab3b1604f09e9c6712aba55647c6c61e9..3f074c4d57354acbeaa20c8b00b56a84e93a1733 100644 --- a/clang/test/CXX/drs/dr16xx.cpp +++ b/clang/test/CXX/drs/dr16xx.cpp @@ -1,12 +1,14 @@ -// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2a -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors - -#if __cplusplus < 201103L -// expected-error@+1 {{variadic macro}} +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected,cxx98-14,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors + +#if __cplusplus == 199711L #define static_assert(...) __extension__ _Static_assert(__VA_ARGS__) +// cxx98-error@-1 {{variadic macros are a C99 feature}} #endif #if __cplusplus >= 201103L @@ -25,9 +27,7 @@ namespace std { namespace dr1601 { // dr1601: 10 enum E : char { e }; -#if __cplusplus < 201103L - // expected-error@-2 {{enumeration types with a fixed underlying type are a C++11 extension}} -#endif +// cxx98-error@-1 {{enumeration types with a fixed underlying type are a C++11 extension}} void f(char); void f(int); void g() { @@ -53,26 +53,30 @@ namespace dr1631 { // dr1631: 3.7 void f(int, A); void test() { - f({0}, {{1}}); // expected-warning {{braces around scalar init}} + f({0}, {{1}}); + // since-cxx11-warning@-1 {{braces around scalar initializer}} } namespace with_error { void f(B, int); // TODO: expected- note {{candidate function}} - void f(int, A); // expected-note {{candidate function}} - void f(int, A, int = 0); // expected-note {{candidate function}} + void f(int, A); // #dr1631-f + void f(int, A, int = 0); // #dr1631-f-int void test() { - f({0}, {{1}}); // expected-error{{call to 'f' is ambiguous}} + f({0}, {{1}}); + // since-cxx11-error@-1 {{call to 'f' is ambiguous}} + // since-cxx11-note@#dr1631-f {{candidate function}} + // since-cxx11-note@#dr1631-f-int {{candidate function}} } } #endif } -namespace dr1638 { // dr1638: yes +namespace dr1638 { // dr1638: 3.1 #if __cplusplus >= 201103L template struct A { - enum class E; // expected-note {{previous}} - enum class F : T; // expected-note 2{{previous}} + enum class E; // #dr1638-E + enum class F : T; // #dr1638-F }; template<> enum class A::E; @@ -83,16 +87,27 @@ namespace dr1638 { // dr1638: yes template<> enum class A::E : int; template<> enum class A::E : int {}; - template<> enum class A::F; // expected-error {{different underlying type}} - template<> enum class A::E : char; // expected-error {{different underlying type}} - template<> enum class A::F : int; // expected-error {{different underlying type}} - - enum class A::E; // expected-error {{template specialization requires 'template<>'}} expected-error {{nested name specifier}} - template enum class A::E; // expected-error {{enumerations cannot be explicitly instantiated}} - enum class A::E *e; // expected-error {{must use 'enum' not 'enum class'}} + template<> enum class A::F; + // since-cxx11-error@-1 {{enumeration redeclared with different underlying type 'int' (was 'short')}} + // since-cxx11-note@#dr1638-F {{previous declaration is here}} + template<> enum class A::E : char; + // since-cxx11-error@-1 {{enumeration redeclared with different underlying type 'char' (was 'int')}} + // since-cxx11-note@#dr1638-E {{previous declaration is here}} + template<> enum class A::F : int; + // since-cxx11-error@-1 {{enumeration redeclared with different underlying type 'int' (was 'char')}} + // since-cxx11-note@#dr1638-F {{previous declaration is here}} + + enum class A::E; + // since-cxx11-error@-1 {{template specialization requires 'template<>'}} + // since-cxx11-error@-2 {{forward declaration of enum class cannot have a nested name specifier}} + template enum class A::E; + // since-cxx11-error@-1 {{enumerations cannot be explicitly instantiated}} + enum class A::E *e; + // since-cxx11-error@-1 {{reference to enumeration must use 'enum' not 'enum class'}} struct B { - friend enum class A::E; // expected-error {{must use 'enum' not 'enum class'}} + friend enum class A::E; + // since-cxx11-error@-1 {{reference to enumeration must use 'enum' not 'enum class'}} }; #endif } @@ -100,37 +115,50 @@ namespace dr1638 { // dr1638: yes namespace dr1645 { // dr1645: 3.9 #if __cplusplus >= 201103L struct A { - constexpr A(int, float = 0); // expected-note {{candidate}} - explicit A(int, int = 0); // expected-note 2{{candidate}} - A(int, int, int = 0) = delete; // expected-note {{candidate}} + constexpr A(int, float = 0); // #dr1645-int-float + explicit A(int, int = 0); // #dr1645-int-int + A(int, int, int = 0) = delete; // #dr1645-int-int-int }; struct B : A { - using A::A; // expected-note 4{{inherited here}} + using A::A; // #dr1645-using }; - constexpr B a(0); // expected-error {{ambiguous}} - constexpr B b(0, 0); // expected-error {{ambiguous}} + constexpr B a(0); + // since-cxx11-error@-1 {{call to constructor of 'const B' is ambiguous}} + // since-cxx11-note@#dr1645-int-float {{candidate inherited constructor}} + // since-cxx11-note@#dr1645-using {{constructor from base class 'A' inherited here}} + // since-cxx11-note@#dr1645-int-int {{candidate inherited constructor}} + // since-cxx11-note@#dr1645-using {{constructor from base class 'A' inherited here}} + constexpr B b(0, 0); + // since-cxx11-error@-1 {{call to constructor of 'const B' is ambiguous}} + // since-cxx11-note@#dr1645-int-int {{candidate inherited constructor}} + // since-cxx11-note@#dr1645-using {{constructor from base class 'A' inherited here}} + // since-cxx11-note@#dr1645-int-int-int {{candidate inherited constructor has been explicitly deleted}} + // since-cxx11-note@#dr1645-using {{constructor from base class 'A' inherited here}} #endif } namespace dr1652 { // dr1652: 3.6 int a, b; - int arr[&a + 1 == &b ? 1 : 2]; // expected-error 2{{variable length array}} - // expected-note@-1 {{points past the end}} + int arr[&a + 1 == &b ? 1 : 2]; + // expected-error@-1 {{variable length arrays in C++ are a Clang extension}} + // expected-note@-2 {{comparison against pointer '&a + 1' that points past the end of a complete object has unspecified value}} + // expected-error@-3 {{variable length array declaration not allowed at file scope}} } namespace dr1653 { // dr1653: 4 c++17 void f(bool b) { ++b; + // cxx98-14-warning@-1 {{incrementing expression of type bool is deprecated and incompatible with C++17}} + // since-cxx17-error@-2 {{SO C++17 does not allow incrementing expression of type bool}} b++; -#if __cplusplus <= 201402L - // expected-warning@-3 {{deprecated}} expected-warning@-2 {{deprecated}} -#else - // expected-error@-5 {{incrementing expression of type bool}} expected-error@-4 {{incrementing expression of type bool}} -#endif - --b; // expected-error {{cannot decrement expression of type bool}} - b--; // expected-error {{cannot decrement expression of type bool}} + // cxx98-14-warning@-1 {{incrementing expression of type bool is deprecated and incompatible with C++17}} + // since-cxx17-error@-2 {{SO C++17 does not allow incrementing expression of type bool}} + --b; + // expected-error@-1 {{cannot decrement expression of type bool}} + b--; + // expected-error@-1 {{cannot decrement expression of type bool}} b += 1; // ok b -= 1; // ok } @@ -138,71 +166,88 @@ namespace dr1653 { // dr1653: 4 c++17 namespace dr1658 { // dr1658: 5 namespace DefCtor { - class A { A(); }; // expected-note 0-2{{here}} - class B { ~B(); }; // expected-note 0-2{{here}} + class A { A(); }; // #dr1658-A1 + class B { ~B(); }; // #dr1658-B1 // The stars align! An abstract class does not construct its virtual bases. struct C : virtual A { C(); virtual void foo() = 0; }; - C::C() = default; // ok, not deleted, expected-error 0-1{{extension}} + C::C() = default; // ok, not deleted + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} struct D : virtual B { D(); virtual void foo() = 0; }; - D::D() = default; // ok, not deleted, expected-error 0-1{{extension}} + D::D() = default; // ok, not deleted + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} // In all other cases, we are not so lucky. - struct E : A { E(); virtual void foo() = 0; }; -#if __cplusplus < 201103L - E::E() = default; // expected-error {{private default constructor}} expected-error {{extension}} expected-note {{here}} -#else - E::E() = default; // expected-error {{would delete}} expected-note@-4{{inaccessible default constructor}} -#endif - struct F : virtual A { F(); }; -#if __cplusplus < 201103L - F::F() = default; // expected-error {{private default constructor}} expected-error {{extension}} expected-note {{here}} -#else - F::F() = default; // expected-error {{would delete}} expected-note@-4{{inaccessible default constructor}} -#endif - - struct G : B { G(); virtual void foo() = 0; }; -#if __cplusplus < 201103L - G::G() = default; // expected-error@-2 {{private destructor}} expected-error {{extension}} expected-note {{here}} -#else - G::G() = default; // expected-error {{would delete}} expected-note@-4{{inaccessible destructor}} -#endif - struct H : virtual B { H(); }; -#if __cplusplus < 201103L - H::H() = default; // expected-error@-2 {{private destructor}} expected-error {{extension}} expected-note {{here}} -#else - H::H() = default; // expected-error {{would delete}} expected-note@-4{{inaccessible destructor}} -#endif + struct E : A { E(); virtual void foo() = 0; }; // #dr1658-E1 + E::E() = default; // #dr1658-E1-ctor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-2 {{base class 'A' has private default constructor}} + // cxx98-note@-3 {{in defaulted default constructor for 'dr1658::DefCtor::E' first required here}} + // cxx98-note@#dr1658-A1 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-E1-ctor {{defaulting this default constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-E1 {{default constructor of 'E' is implicitly deleted because base class 'A' has an inaccessible default constructor}} + struct F : virtual A { F(); }; // #dr1658-F1 + F::F() = default; // #dr1658-F1-ctor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-2 {{inherited virtual base class 'A' has private default constructor}} + // cxx98-note@-3 {{in defaulted default constructor for 'dr1658::DefCtor::F' first required here}} + // cxx98-note@#dr1658-A1 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-F1-ctor {{defaulting this default constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-F1 {{default constructor of 'F' is implicitly deleted because base class 'A' has an inaccessible default constructor}} + + struct G : B { G(); virtual void foo() = 0; }; // #dr1658-G1 + G::G() = default; // #dr1658-G1-ctor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@#dr1658-G1 {{base class 'B' has private destructor}} + // cxx98-note@#dr1658-G1-ctor {{in defaulted default constructor for 'dr1658::DefCtor::G' first required here}} + // cxx98-note@#dr1658-B1 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-G1-ctor {{defaulting this default constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-G1 {{default constructor of 'G' is implicitly deleted because base class 'B' has an inaccessible destructor}} + struct H : virtual B { H(); }; // #dr1658-H1 + H::H() = default; // #dr1658-H1-ctor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@#dr1658-H1 {{base class 'B' has private destructor}} + // cxx98-note@#dr1658-H1-ctor {{in defaulted default constructor for 'dr1658::DefCtor::H' first required here}} + // cxx98-note@#dr1658-B1 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-H1-ctor {{defaulting this default constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-H1 {{default constructor of 'H' is implicitly deleted because base class 'B' has an inaccessible destructor}} } namespace Dtor { - class B { ~B(); }; // expected-note 0-2{{here}} + class B { ~B(); }; // #dr1658-B2 struct D : virtual B { ~D(); virtual void foo() = 0; }; - D::~D() = default; // ok, not deleted, expected-error 0-1{{extension}} - - struct G : B { ~G(); virtual void foo() = 0; }; -#if __cplusplus < 201103L - G::~G() = default; // expected-error@-2 {{private destructor}} expected-error {{extension}} expected-note {{here}} -#else - G::~G() = default; // expected-error {{would delete}} expected-note@-4{{inaccessible destructor}} -#endif - struct H : virtual B { ~H(); }; -#if __cplusplus < 201103L - H::~H() = default; // expected-error@-2 {{private destructor}} expected-error {{extension}} expected-note {{here}} -#else - H::~H() = default; // expected-error {{would delete}} expected-note@-4{{inaccessible destructor}} -#endif + D::~D() = default; // ok, not deleted + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + + struct G : B { ~G(); virtual void foo() = 0; }; // #dr1658-G2 + G::~G() = default; // #dr1658-G2-dtor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@#dr1658-G2 {{base class 'B' has private destructor}} + // cxx98-note@#dr1658-G2-dtor {{in defaulted destructor for 'dr1658::Dtor::G' first required here}} + // cxx98-note@#dr1658-B2 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-G2-dtor {{defaulting this destructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-G2 {{destructor of 'G' is implicitly deleted because base class 'B' has an inaccessible destructor}} + struct H : virtual B { ~H(); }; // #dr1658-H2 + H::~H() = default; // #dr1658-H2-dtor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@#dr1658-H2 {{base class 'B' has private destructor}} + // cxx98-note@#dr1658-H2-dtor {{in defaulted destructor for 'dr1658::Dtor::H' first required here}} + // cxx98-note@#dr1658-B2 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-H2-dtor {{defaulting this destructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-H2 {{destructor of 'H' is implicitly deleted because base class 'B' has an inaccessible destructor}} } namespace MemInit { - struct A { A(int); }; // expected-note {{here}} + struct A { A(int); }; // #dr1658-A3 struct B : virtual A { B() {} virtual void f() = 0; }; struct C : virtual A { - C() {} // expected-error {{must explicitly initialize}} + C() {} + // expected-error@-1 {{constructor for 'dr1658::MemInit::C' must explicitly initialize the base class 'A' which does not have a default constructor}} + // expected-note@#dr1658-A3 {{'dr1658::MemInit::A' declared here}} }; } @@ -220,28 +265,51 @@ namespace dr1658 { // dr1658: 5 } namespace CopyCtor { - class A { A(const A&); A(A&&); }; // expected-note 0-4{{here}} expected-error 0-1{{extension}} - - struct C : virtual A { C(const C&); C(C&&); virtual void foo() = 0; }; // expected-error 0-1{{extension}} - C::C(const C&) = default; // expected-error 0-1{{extension}} - C::C(C&&) = default; // expected-error 0-2{{extension}} - - struct E : A { E(const E&); E(E&&); virtual void foo() = 0; }; // expected-error 0-1{{extension}} -#if __cplusplus < 201103L - E::E(const E&) = default; // expected-error {{private copy constructor}} expected-error {{extension}} expected-note {{here}} - E::E(E&&) = default; // expected-error {{private move constructor}} expected-error 2{{extension}} expected-note {{here}} -#else - E::E(const E&) = default; // expected-error {{would delete}} expected-note@-5{{inaccessible copy constructor}} - E::E(E&&) = default; // expected-error {{would delete}} expected-note@-6{{inaccessible move constructor}} -#endif - struct F : virtual A { F(const F&); F(F&&); }; // expected-error 0-1{{extension}} -#if __cplusplus < 201103L - F::F(const F&) = default; // expected-error {{private copy constructor}} expected-error {{extension}} expected-note {{here}} - F::F(F&&) = default; // expected-error {{private move constructor}} expected-error 2{{extension}} expected-note {{here}} -#else - F::F(const F&) = default; // expected-error {{would delete}} expected-note@-5{{inaccessible copy constructor}} - F::F(F&&) = default; // expected-error {{would delete}} expected-note@-6{{inaccessible move constructor}} -#endif + class A { A(const A&); A(A&&); }; // #dr1658-A5 + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + + struct C : virtual A { C(const C&); C(C&&); virtual void foo() = 0; }; + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + C::C(const C&) = default; + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + C::C(C&&) = default; + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + // cxx98-error@-2 {{defaulted function definitions are a C++11 extension}} + + struct E : A { E(const E&); E(E&&); virtual void foo() = 0; }; // #dr1658-E5 + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + E::E(const E&) = default; // #dr1658-E5-copy-ctor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-2 {{base class 'A' has private copy constructor}} + // cxx98-note@-3 {{in defaulted copy constructor for 'dr1658::CopyCtor::E' first required here}} + // cxx98-note@#dr1658-A5 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-E5-copy-ctor {{defaulting this copy constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-E5 {{copy constructor of 'E' is implicitly deleted because base class 'A' has an inaccessible copy constructor}} + E::E(E&&) = default; // #dr1658-E5-move-ctor + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + // cxx98-error@-2 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-3 {{base class 'A' has private move constructor}} + // cxx98-note@-4 {{in defaulted move constructor for 'dr1658::CopyCtor::E' first required here}} + // cxx98-note@#dr1658-A5 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-E5-move-ctor {{defaulting this move constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-E5 {{move constructor of 'E' is implicitly deleted because base class 'A' has an inaccessible move constructor}} + struct F : virtual A { F(const F&); F(F&&); }; // #dr1658-F5 + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + F::F(const F&) = default; // #dr1658-F5-copy-ctor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-2 {{inherited virtual base class 'A' has private copy constructor}} + // cxx98-note@-3 {{in defaulted copy constructor for 'dr1658::CopyCtor::F' first required here}} + // cxx98-note@#dr1658-A5 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-F5-copy-ctor {{defaulting this copy constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-F5 {{copy constructor of 'F' is implicitly deleted because base class 'A' has an inaccessible copy constructor}} + F::F(F&&) = default; // #dr1658-F5-move-ctor + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + // cxx98-error@-2 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-3 {{inherited virtual base class 'A' has private move constructor}} + // cxx98-note@-4 {{in defaulted move constructor for 'dr1658::CopyCtor::F' first required here}} + // cxx98-note@#dr1658-A5 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-F5-move-ctor {{defaulting this move constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-F5 {{move constructor of 'F' is implicitly deleted because base class 'A' has an inaccessible move constructor}} } // assignment case is superseded by dr2180 @@ -274,31 +342,38 @@ namespace dr1672 { // dr1672: 7 namespace dr1684 { // dr1684: 3.6 #if __cplusplus >= 201103L - struct NonLiteral { // expected-note {{because}} + struct NonLiteral { // #dr1684-struct NonLiteral(); - constexpr int f() { return 0; } // expected-warning 0-1{{will not be implicitly 'const'}} + constexpr int f() { return 0; } + // cxx11-warning@-1 {{'constexpr' non-static member function will not be implicitly 'const' in C++14; add 'const' to avoid a change in behavior}} }; constexpr int f(NonLiteral &) { return 0; } - constexpr int f(NonLiteral) { return 0; } // expected-error {{not a literal type}} + constexpr int f(NonLiteral) { return 0; } + // since-cxx11-error@-1 {{constexpr function's 1st parameter type 'NonLiteral' is not a literal type}} + // since-cxx11-note@#dr1684-struct {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} #endif } namespace dr1687 { // dr1687: 7 template struct To { - operator T(); // expected-note 2{{first operand was implicitly converted to type 'int *'}} - // expected-note@-1 {{second operand was implicitly converted to type 'double'}} -#if __cplusplus > 201703L - // expected-note@-3 2{{operand was implicitly converted to type 'dr1687::E}} -#endif + operator T(); // #dr1687-op-T }; - int *a = To() + 100.0; // expected-error {{invalid operands to binary expression ('To' and 'double')}} - int *b = To() + To(); // expected-error {{invalid operands to binary expression ('To' and 'To')}} + int *a = To() + 100.0; + // expected-error@-1 {{invalid operands to binary expression ('To' and 'double')}} + // expected-note@#dr1687-op-T {{first operand was implicitly converted to type 'int *'}} + // since-cxx20-note@#dr1687-op-T {{second operand was implicitly converted to type 'dr1687::E2'}} + int *b = To() + To(); + // expected-error@-1 {{invalid operands to binary expression ('To' and 'To')}} + // expected-note@#dr1687-op-T {{first operand was implicitly converted to type 'int *'}} + // expected-note@#dr1687-op-T {{second operand was implicitly converted to type 'double'}} -#if __cplusplus > 201703L +#if __cplusplus >= 202002L enum E1 {}; enum E2 {}; - auto c = To() <=> To(); // expected-error {{invalid operands to binary expression ('To' and 'To')}} + auto c = To() <=> To(); + // since-cxx20-error@-1 {{invalid operands to binary expression ('To' and 'To')}} + // since-cxx20-note@#dr1687-op-T {{operand was implicitly converted to type 'dr1687::E}} #endif } @@ -325,12 +400,14 @@ namespace dr1691 { // dr1691: 9 void f(E); } enum M::E : int {}; - void g(M::E); // expected-note {{declared here}} + void g(M::E); // #dr1691-g } void test() { N::M::E e; f(e); // ok - g(e); // expected-error {{use of undeclared}} + g(e); + // since-cxx11-error@-1 {{use of undeclared identifier 'g'; did you mean 'N::g'?}} + // since-cxx11-note@#dr1691-g {{'N::g' declared here}} } #endif } @@ -356,7 +433,9 @@ namespace dr1696 { // dr1696: 7 extern struct A a; struct A { const A &x = { A{a, a} }; - const A &y = { A{} }; // expected-error {{default member initializer for 'y' needed within definition of enclosing class 'A' outside of member functions}} expected-note {{here}} + const A &y = { A{} }; + // since-cxx14-error@-1 {{default member initializer for 'y' needed within definition of enclosing class 'A' outside of member functions}} + // since-cxx14-note@-2 {{default member initializer declared here}} }; A a{a, a}; #endif @@ -365,16 +444,20 @@ namespace dr1696 { // dr1696: 7 struct A { A(); ~A(); }; #if __cplusplus >= 201103L struct B { - A &&a; // expected-note {{declared here}} - B() : a{} {} // expected-error {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + A &&a; // #dr1696-a + B() : a{} {} + // since-cxx11-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx11-note@#dr1696-a {{reference member declared here}} } b; #endif struct C { C(); - const A &a; // expected-note {{declared here}} + const A &a; // #dr1696-C-a }; - C::C() : a(A()) {} // expected-error {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + C::C() : a(A()) {} + // expected-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // expected-note@#dr1696-C-a {{reference member declared here}} #if __cplusplus >= 201103L // This is OK in C++14 onwards, per DR1815, though we don't support that yet: @@ -383,51 +466,62 @@ namespace dr1696 { // dr1696: 7 // D1 d1 = {A()}; // ... which lifetime-extends the A temporary. struct D1 { -#if __cplusplus < 201402L - // expected-error@-2 {{binds to a temporary}} -#endif - const A &a = A(); // expected-note {{default member init}} + // cxx11-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // cxx11-note@#dr1696-d1 {{in implicit default constructor for 'dr1696::D1' first required here}} + // cxx11-note@#dr1696-D1-a {{initializing field 'a' with default member initializer}} + const A &a = A(); // #dr1696-D1-a }; - D1 d1 = {}; -#if __cplusplus < 201402L - // expected-note@-2 {{first required here}} -#else - // expected-warning-re@-4 {{sorry, lifetime extension {{.*}} not supported}} -#endif + D1 d1 = {}; // #dr1696-d1 + // since-cxx14-warning@-1 {{sorry, lifetime extension of temporary created by aggregate initialization using default member initializer is not supported; lifetime of temporary will end at the end of the full-expression}} + // since-cxx14-note@#dr1696-D1-a {{initializing field 'a' with default member initializer}} struct D2 { - const A &a = A(); // expected-note {{default member init}} - D2() {} // expected-error {{binds to a temporary}} + const A &a = A(); // #dr1696-D2-a + D2() {} + // since-cxx11-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx11-note@#dr1696-D2-a {{initializing field 'a' with default member initializer}} }; - struct D3 { // expected-error {{binds to a temporary}} - const A &a = A(); // expected-note {{default member init}} + struct D3 { + // since-cxx11-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx11-note@#dr1696-d3 {{in implicit default constructor for 'dr1696::D3' first required here}} + // since-cxx11-note@#dr1696-D3-a {{initializing field 'a' with default member initializer}} + const A &a = A(); // #dr1696-D3-a }; - D3 d3; // expected-note {{first required here}} + D3 d3; // #dr1696-d3 struct haslist1 { - std::initializer_list il; // expected-note {{'std::initializer_list' member}} - haslist1(int i) : il{i, 2, 3} {} // expected-error {{backing array for 'std::initializer_list' member 'il' is a temporary object}} + std::initializer_list il; // #dr1696-il-1 + haslist1(int i) : il{i, 2, 3} {} + // since-cxx11-error@-1 {{backing array for 'std::initializer_list' member 'il' is a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx11-note@#dr1696-il-1 {{'std::initializer_list' member declared here}} }; struct haslist2 { - std::initializer_list il; // expected-note {{'std::initializer_list' member}} + std::initializer_list il; // #dr1696-il-2 haslist2(); }; - haslist2::haslist2() : il{1, 2} {} // expected-error {{backing array for 'std::initializer_list' member 'il' is a temporary object}} + haslist2::haslist2() : il{1, 2} {} + // since-cxx11-error@-1 {{backing array for 'std::initializer_list' member 'il' is a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx11-note@#dr1696-il-2 {{'std::initializer_list' member declared here}} struct haslist3 { std::initializer_list il = {1, 2, 3}; }; - struct haslist4 { // expected-error {{backing array for 'std::initializer_list' member 'il' is a temporary object}} - std::initializer_list il = {1, 2, 3}; // expected-note {{default member initializer}} + struct haslist4 { + // since-cxx11-error@-1 {{backing array for 'std::initializer_list' member 'il' is a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx11-note@#dr1696-hl4 {{in implicit default constructor for 'dr1696::haslist4' first required here}} + // since-cxx11-note@#dr1696-il-4 {{initializing field 'il' with default member initializer}} + std::initializer_list il = {1, 2, 3}; // #dr1696-il-4 }; - haslist4 hl4; // expected-note {{in implicit default constructor}} + haslist4 hl4; // #dr1696-hl4 struct haslist5 { - std::initializer_list il = {1, 2, 3}; // expected-note {{default member initializer}} - haslist5() {} // expected-error {{backing array for 'std::initializer_list' member 'il' is a temporary object}} + std::initializer_list il = {1, 2, 3}; // #dr1696-il-5 + haslist5() {} + // since-cxx11-error@-1 {{backing array for 'std::initializer_list' member 'il' is a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx11-note@#dr1696-il-5 {{nitializing field 'il' with default member initializer}} }; #endif } diff --git a/clang/test/CXX/drs/dr17xx.cpp b/clang/test/CXX/drs/dr17xx.cpp index 219119d1a4cd087368e4dad08e395257e50e991d..0c44fb231ce51a7d6b5f1634f3451c8b4ebef60a 100644 --- a/clang/test/CXX/drs/dr17xx.cpp +++ b/clang/test/CXX/drs/dr17xx.cpp @@ -1,10 +1,10 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2c %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors namespace dr1710 { // dr1710: no // FIXME: all of the following is well-formed @@ -32,13 +32,17 @@ namespace dr1715 { // dr1715: 3.9 struct D : B { using B::B; }; - struct E : B { // expected-note 2{{candidate}} - template E(T t, typename T::Q q) : B(t, q) {} // expected-note {{'Q' is a private member}} + struct E : B { // #dr1715-E + template E(T t, typename T::Q q) : B(t, q) {} // #dr1715-E-ctor }; B b(S(), 1); D d(S(), 2); - E e(S(), 3); // expected-error {{no match}} + E e(S(), 3); + // since-cxx11-error@-1 {{no matching constructor for initialization of 'E'}} + // since-cxx11-note@#dr1715-E-ctor {{candidate template ignored: substitution failure [with T = S]: 'Q' is a private member of 'dr1715::S'}} + // since-cxx11-note@#dr1715-E {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}} + // since-cxx11-note@#dr1715-E {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}} #endif } @@ -73,12 +77,15 @@ struct S { struct L : S { using S::S; }; - typename T::type value; // expected-error {{no member}} - L l(value); // expected-note {{instantiation of}} + typename T::type value; + // since-cxx11-error@-1 {{type 'int' cannot be used prior to '::' because it has no members}} + // since-cxx11-note@#dr1736-l {{in instantiation of function template specialization 'dr1736::S::S' requested here}} + // since-cxx11-note@#dr1736-s {{in instantiation of function template specialization 'dr1736::S::S' requested here}} + L l(value); // #dr1736-l } }; struct Q { typedef int type; } q; -S s(q); // expected-note {{instantiation of}} +S s(q); // #dr1736-s #endif } @@ -91,18 +98,23 @@ namespace dr1753 { // dr1753: 11 n.~T(); n.T::~T(); - n.dr1753::~T(); // expected-error {{'dr1753' does not refer to a type name in pseudo-destructor}} + n.dr1753::~T(); + // expected-error@-1 {{'dr1753' does not refer to a type name in pseudo-destructor expression; expected the name of type 'T' (aka 'int')}} n.dr1753::T::~T(); - n.A::~T(); // expected-error {{the type of object expression ('T' (aka 'int')) does not match the type being destroyed ('A') in pseudo-destructor expression}} + n.A::~T(); + // expected-error@-1 {{the type of object expression ('T' (aka 'int')) does not match the type being destroyed ('A') in pseudo-destructor expression}} n.A::T::~T(); - n.B::~T(); // expected-error {{'B' does not refer to a type name in pseudo-destructor expression}} + n.B::~T(); + // expected-error@-1 {{'B' does not refer to a type name in pseudo-destructor expression; expected the name of type 'T' (aka 'int')}} n.B::T::~T(); #if __cplusplus >= 201103L - n.decltype(n)::~T(); // expected-error {{not a class, namespace, or enumeration}} - n.T::~decltype(n)(); // expected-error {{expected a class name after '~'}} + n.decltype(n)::~T(); + // since-cxx11-error@-1 {{'decltype(n)' (aka 'int') is not a class, namespace, or enumeration}} + n.T::~decltype(n)(); + // since-cxx11-error@-1 {{expected a class name after '~' to name a destructor}} n.~decltype(n)(); // OK #endif } @@ -141,9 +153,9 @@ namespace dr1758 { // dr1758: 3.7 namespace dr1762 { // dr1762: 14 #if __cplusplus >= 201103L float operator ""_E(const char *); - // expected-error@+2 {{invalid suffix on literal; C++11 requires a space between literal and identifier}} - // expected-warning@+1 {{user-defined literal suffixes not starting with '_' are reserved; no literal will invoke this operator}} float operator ""E(const char *); + // since-cxx11-error@-1 {{invalid suffix on literal; C++11 requires a space between literal and identifier}} + // since-cxx11-warning@-2 {{user-defined literal suffixes not starting with '_' are reserved; no literal will invoke this operator}} #endif } diff --git a/clang/test/CXX/drs/dr18xx.cpp b/clang/test/CXX/drs/dr18xx.cpp index 7ac26737382281e28225ddd271b13d433bb52c0c..fbe67bd0c2f6dbbd79b26e8df4c249a0c23a74a7 100644 --- a/clang/test/CXX/drs/dr18xx.cpp +++ b/clang/test/CXX/drs/dr18xx.cpp @@ -1,14 +1,14 @@ -// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors - -#if __cplusplus < 201103L -// expected-error@+1 {{variadic macro}} +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected,cxx98 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-17,since-cxx11 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-17,since-cxx11,since-cxx14 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-17,since-cxx11,since-cxx14 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx14 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx14 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx14 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors + +#if __cplusplus == 199711L #define static_assert(...) __extension__ _Static_assert(__VA_ARGS__) +// cxx98-error@-1 {{variadic macros are a C99 feature}} #endif namespace dr1812 { // dr1812: no @@ -16,7 +16,7 @@ namespace dr1812 { // dr1812: no #if __cplusplus >= 201103L template struct A { using B = typename T::C; - // expected-error@-1 {{use 'template' keyword to treat 'C' as a dependent template name}} + // since-cxx11-error@-1 {{use 'template' keyword to treat 'C' as a dependent template name}} }; #endif } // namespace dr1812 @@ -54,15 +54,20 @@ namespace dr1814 { // dr1814: yes namespace dr1815 { // dr1815: no #if __cplusplus >= 201402L // FIXME: needs codegen test - struct A { int &&r = 0; }; // expected-note {{default member init}} - A a = {}; // FIXME expected-warning {{not supported}} - - struct B { int &&r = 0; }; // expected-error {{binds to a temporary}} expected-note {{default member init}} - B b; // expected-note {{here}} + struct A { int &&r = 0; }; // #dr1815-A + A a = {}; + // since-cxx14-warning@-1 {{sorry, lifetime extension of temporary created by aggregate initialization using default member initializer is not supported; lifetime of temporary will end at the end of the full-expression}} FIXME + // since-cxx14-note@#dr1815-A {{initializing field 'r' with default member initializer}} + + struct B { int &&r = 0; }; // #dr1815-B + // since-cxx14-error@-1 {{reference member 'r' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx14-note@#dr1815-B {{initializing field 'r' with default member initializer}} + // since-cxx14-note@#dr1815-b {{in implicit default constructor for 'dr1815::B' first required here}} + B b; // #dr1815-b #endif } -namespace dr1821 { // dr1821: yes +namespace dr1821 { // dr1821: 2.9 struct A { template struct B { void f(); @@ -80,9 +85,9 @@ struct A { namespace dr1822 { // dr1822: yes #if __cplusplus >= 201103L - int a; + double a; auto x = [] (int a) { -#pragma clang __debug dump a // CHECK: ParmVarDecl + static_assert(__is_same(decltype(a), int), "should be resolved to lambda parameter"); }; #endif } @@ -98,16 +103,22 @@ namespace dr1837 { // dr1837: 3.3 }; class Outer { - friend auto Other::q() -> decltype(this->p()) *; // expected-error {{invalid use of 'this'}} + friend auto Other::q() -> decltype(this->p()) *; + // since-cxx11-error@-1 {{invalid use of 'this' outside of a non-static member function}} int g(); int f() { extern void f(decltype(this->g()) *); struct Inner { - static_assert(Fishg())>::value, ""); // expected-error {{invalid use of 'this'}} - enum { X = Fishf())>::value }; // expected-error {{invalid use of 'this'}} - struct Inner2 : Fishg())> { }; // expected-error {{invalid use of 'this'}} - friend void f(decltype(this->g()) *); // expected-error {{invalid use of 'this'}} - friend auto Other::q() -> decltype(this->p()) *; // expected-error {{invalid use of 'this'}} + static_assert(Fishg())>::value, ""); + // since-cxx11-error@-1 {{invalid use of 'this' outside of a non-static member function}} + enum { X = Fishf())>::value }; + // since-cxx11-error@-1 {{invalid use of 'this' outside of a non-static member function}} + struct Inner2 : Fishg())> { }; + // since-cxx11-error@-1 {{invalid use of 'this' outside of a non-static member function}} + friend void f(decltype(this->g()) *); + // since-cxx11-error@-1 {{invalid use of 'this' outside of a non-static member function}} + friend auto Other::q() -> decltype(this->p()) *; + // since-cxx11-error@-1 {{invalid use of 'this' outside of a non-static member function}} }; return 0; } @@ -135,19 +146,21 @@ namespace dr1872 { // dr1872: 9 constexpr int x = A().f(); constexpr int y = A().f(); -#if __cplusplus <= 201703L - // expected-error@-2 {{constant expression}} expected-note@-2 {{call to virtual function}} -#else + // cxx11-17-error@-1 {{constexpr variable 'y' must be initialized by a constant expression}} + // cxx11-17-note@-2 {{cannot evaluate call to virtual function in a constant expression in C++ standards before C++20}} +#if __cplusplus >= 202002L static_assert(y == 0); #endif // Note, this is invalid even though it would not use virtual dispatch. constexpr int y2 = A().A::f(); -#if __cplusplus <= 201703L - // expected-error@-2 {{constant expression}} expected-note@-2 {{call to virtual function}} -#else + // cxx11-17-error@-1 {{constexpr variable 'y2' must be initialized by a constant expression}} + // cxx11-17-note@-2 {{cannot evaluate call to virtual function in a constant expression in C++ standards before C++20}} +#if __cplusplus >= 202002L static_assert(y == 0); #endif - constexpr int z = A().f(); // expected-error {{constant expression}} expected-note {{non-literal type}} + constexpr int z = A().f(); + // since-cxx11-error@-1 {{constexpr variable 'z' must be initialized by a constant expression}} + // since-cxx11-note@-2 {{non-literal type 'A' cannot be used in a constant expression}} #endif } @@ -166,33 +179,38 @@ namespace dr1881 { // dr1881: 7 void dr1891() { // dr1891: 4 #if __cplusplus >= 201103L int n; - auto a = []{}; // expected-note 0-4{{}} - auto b = [=]{ return n; }; // expected-note 0-4{{}} + auto a = []{}; // #dr1891-a + auto b = [=]{ return n; }; // #dr1891-b typedef decltype(a) A; typedef decltype(b) B; static_assert(!__has_trivial_constructor(A), ""); -#if __cplusplus > 201703L - // expected-error@-2 {{failed}} -#endif + // since-cxx20-error@-1 {{failed}} static_assert(!__has_trivial_constructor(B), ""); // C++20 allows default construction for non-capturing lambdas (P0624R2). A x; -#if __cplusplus <= 201703L - // expected-error@-2 {{no matching constructor}} -#endif - B y; // expected-error {{no matching constructor}} + // cxx11-17-error@-1 {{no matching constructor for initialization of 'A' (aka '(lambda at}} + // cxx11-17-note@#dr1891-a {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}} + // cxx11-17-note@#dr1891-a {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}} + B y; + // since-cxx11-error@-1 {{no matching constructor for initialization of 'B' (aka '(lambda at}} + // since-cxx11-note@#dr1891-b {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}} + // since-cxx11-note@#dr1891-b {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}} // C++20 allows assignment for non-capturing lambdas (P0624R2). a = a; + // cxx11-17-error-re@-1 {{{{object of type '\(lambda at .+\)' cannot be assigned because its copy assignment operator is implicitly deleted}}}} + // cxx11-17-note@#dr1891-a {{lambda expression begins here}} a = static_cast(a); -#if __cplusplus <= 201703L - // expected-error@-3 {{copy assignment operator is implicitly deleted}} - // expected-error@-3 {{copy assignment operator is implicitly deleted}} -#endif - b = b; // expected-error {{copy assignment operator is implicitly deleted}} - b = static_cast(b); // expected-error {{copy assignment operator is implicitly deleted}} + // cxx11-17-error-re@-1 {{{{object of type '\(lambda at .+\)' cannot be assigned because its copy assignment operator is implicitly deleted}}}} + // cxx11-17-note@#dr1891-a {{lambda expression begins here}} + b = b; + // since-cxx11-error-re@-1 {{{{object of type '\(lambda at .+\)' cannot be assigned because its copy assignment operator is implicitly deleted}}}} + // since-cxx11-note@#dr1891-b {{lambda expression begins here}} + b = static_cast(b); + // since-cxx11-error-re@-1 {{{{object of type '\(lambda at .+\)' cannot be assigned because its copy assignment operator is implicitly deleted}}}} + // since-cxx11-note@#dr1891-b {{lambda expression begins here}} #endif } diff --git a/clang/test/CXX/drs/dr19xx.cpp b/clang/test/CXX/drs/dr19xx.cpp index b15be762ecd1099b5b2b1eb7b3a309895a9db7b6..716b1476831ed96c37a7255eedcf6faafb1e5e33 100644 --- a/clang/test/CXX/drs/dr19xx.cpp +++ b/clang/test/CXX/drs/dr19xx.cpp @@ -1,41 +1,36 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++1z %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected,cxx98-11,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx98-11,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors namespace std { struct type_info; } namespace dr1902 { // dr1902: 3.7 struct A {}; struct B { - B(A); -#if __cplusplus >= 201103L - // expected-note@-2 {{candidate}} -#endif - - B() = delete; -#if __cplusplus < 201103L - // expected-error@-2 {{extension}} -#endif - - B(const B&) // expected-note {{deleted here}} -#if __cplusplus >= 201103L - // expected-note@-2 {{candidate}} -#else - // expected-error@+2 {{extension}} -#endif - = delete; - + B(A); // #dr1902-B-A + B() = delete; // #dr1902-B-ctor + // cxx98-error@-1 {{deleted function definitions are a C++11 extension}} + B(const B&) = delete; // #dr1902-B-copy-ctor + // cxx98-error@-1 {{deleted function definitions are a C++11 extension}} operator A(); }; extern B b1; - B b2(b1); // expected-error {{call to deleted}} + B b2(b1); + // expected-error@-1 {{call to deleted constructor of 'B'}} + // expected-note@#dr1902-B-copy-ctor {{'B' has been explicitly marked deleted here}} #if __cplusplus >= 201103L // This is ambiguous, even though calling the B(const B&) constructor would // both directly and indirectly call a deleted function. - B b({}); // expected-error {{ambiguous}} + B b({}); + // since-cxx11-error@-1 {{call to constructor of 'B' is ambiguous}} + // since-cxx11-note@#dr1902-B-A {{candidate constructor}} + // since-cxx11-note@#dr1902-B-copy-ctor {{candidate constructor has been explicitly deleted}} #endif } @@ -64,26 +59,33 @@ namespace dr1903 { } } -namespace dr1909 { // dr1909: yes +namespace dr1909 { // dr1909: 3.7 struct A { - template struct A {}; // expected-error {{member 'A' has the same name as its class}} + template struct A {}; + // expected-error@-1 {{member 'A' has the same name as its class}} }; struct B { - template void B() {} // expected-error {{constructor cannot have a return type}} + template void B() {} + // expected-error@-1 {{constructor cannot have a return type}} }; struct C { - template static int C; // expected-error {{member 'C' has the same name as its class}} expected-error 0-1{{extension}} + template static int C; + // expected-error@-1 {{member 'C' has the same name as its class}} + // cxx98-11-error@-2 {{variable templates are a C++14 extension}} }; struct D { - template using D = int; // expected-error {{member 'D' has the same name as its class}} expected-error 0-1{{extension}} + template using D = int; + // cxx98-error@-1 {{alias declarations are a C++11 extension}} + // expected-error@-2 {{member 'D' has the same name as its class}} }; } -namespace dr1940 { // dr1940: yes +namespace dr1940 { // dr1940: 3.5 #if __cplusplus >= 201103L static union { static_assert(true, ""); // ok - static_assert(false, ""); // expected-error {{static assertion failed}} + static_assert(false, ""); + // since-cxx11-error@-1 {{static assertion failed}} int not_empty; }; #endif @@ -119,18 +121,21 @@ derived d2(42, 9); #endif } -namespace dr1947 { // dr1947: yes +namespace dr1947 { // dr1947: 3.5 #if __cplusplus >= 201402L unsigned o = 0'01; // ok -unsigned b = 0b'01; // expected-error {{invalid digit 'b' in octal constant}} -unsigned x = 0x'01; // expected-error {{invalid suffix 'x'01' on integer constant}} +unsigned b = 0b'01; +// since-cxx14-error@-1 {{invalid digit 'b' in octal constant}} +unsigned x = 0x'01; +// since-cxx14-error@-1 {{invalid suffix 'x'01' on integer constant}} #endif } #if __cplusplus >= 201103L -// dr1948: yes +// dr1948: 3.5 // FIXME: This diagnostic could be improved. -void *operator new(__SIZE_TYPE__) noexcept { return nullptr; } // expected-error{{exception specification in declaration does not match previous declaration}} +void *operator new(__SIZE_TYPE__) noexcept { return nullptr; } +// since-cxx11-error@-1 {{exception specification in declaration does not match previous declaration}} #endif namespace dr1959 { // dr1959: 3.9 @@ -139,22 +144,31 @@ namespace dr1959 { // dr1959: 3.9 struct c; struct a { a() = default; - a(const a &) = delete; // expected-note {{deleted}} + a(const a &) = delete; // #dr1959-copy-ctor a(const b &) = delete; // not inherited - a(c &&) = delete; // expected-note {{not viable}} - template a(T) = delete; // expected-note {{would take its own class type by value}} + a(c &&) = delete; // #dr1959-move-ctor + template a(T) = delete; // #dr1959-temp-ctor }; - struct b : a { // expected-note {{cannot bind}} expected-note {{deleted because}} - using a::a; // expected-note 2{{inherited here}} + struct b : a { // #dr1959-b + using a::a; // #dr1959-using-a }; a x; // FIXME: As a resolution to an open DR against P0136R0, we disallow // use of inherited constructors to construct from a single argument // where the base class is reference-related to the argument type. - b y = x; // expected-error {{no viable conversion}} - b z = z; // expected-error {{deleted}} + b y = x; + // since-cxx11-error@-1 {{no viable conversion from 'a' to 'b'}} + // since-cxx11-note@#dr1959-move-ctor {{candidate inherited constructor not viable: no known conversion from 'a' to 'c &&' for 1st argument}} + // since-cxx11-note@#dr1959-using-a {{constructor from base class 'a' inherited here}} + // since-cxx11-note@#dr1959-b {{candidate constructor (the implicit copy constructor) not viable: cannot bind base class object of type 'a' to derived class reference 'const b &' for 1st argument}} + // since-cxx11-note@#dr1959-temp-ctor {{candidate template ignored: instantiation would take its own class type by value}} + // since-cxx11-note@#dr1959-using-a {{constructor from base class 'a' inherited here}} + b z = z; + // since-cxx11-error@-1 {{call to implicitly-deleted copy constructor of 'b'}} + // since-cxx11-note@#dr1959-b {{copy constructor of 'b' is implicitly deleted because base class 'a' has a deleted copy constructor}} + // since-cxx11-note@#dr1959-copy-ctor {{'a' has been explicitly marked deleted here}} struct c : a { using a::a; @@ -191,16 +205,28 @@ using A::g; namespace dr1966 { // dr1966: 11 #if __cplusplus >= 201103L struct A { - enum E : int {1}; // expected-error {{expected identifier}} (not bit-field) + enum E : int {1}; + // since-cxx11-error@-1 {{expected identifier}} (not bit-field) }; - auto *p1 = new enum E : int; // expected-error {{only permitted as a standalone declaration}} - auto *p2 = new enum F : int {}; // expected-error {{only permitted as a standalone declaration}} - auto *p3 = true ? new enum G : int {}; // expected-error {{forward reference}} expected-error {{incomplete}} expected-note {{declaration}} - auto h() -> enum E : int {}; // expected-error {{only permitted as a standalone declaration}} - - enum X : enum Y : int {} {}; // expected-error {{cannot be defined in a type specifier}} + auto *p1 = new enum E : int; + // since-cxx11-error@-1 {{non-defining declaration of enumeration with a fixed underlying type is only permitted as a standalone declaration}} + auto *p2 = new enum F : int {}; + // since-cxx11-error@-1 {{non-defining declaration of enumeration with a fixed underlying type is only permitted as a standalone declaration}} + auto *p3 = true ? new enum G : int {}; + // since-cxx11-error@-1 {{ISO C++ forbids forward references to 'enum' types}} + // since-cxx11-error@-2 {{allocation of incomplete type 'enum G'}} + // since-cxx11-note@-3 {{forward declaration of 'dr1966::G'}} + auto h() -> enum E : int {}; + // since-cxx11-error@-1 {{non-defining declaration of enumeration with a fixed underlying type is only permitted as a standalone declaration}} + + enum X : enum Y : int {} {}; + // since-cxx11-error@-1 {{'dr1966::Y' cannot be defined in a type specifier}} struct Q { - enum X : enum Y : int {} {}; // expected-error +{{}} + // FIXME: can we emit something nicer than that? + enum X : enum Y : int {} {}; + // since-cxx11-error@-1 {{non-defining declaration of enumeration with a fixed underlying type is only permitted as a standalone declaration; missing list of enumerators?}} + // since-cxx11-error@-2 {{non-integral type 'enum Y' is an invalid underlying type}} + // since-cxx11-error@-3 {{anonymous bit-field cannot have a default member initializer}} }; #endif } diff --git a/clang/test/CXX/drs/dr20xx.cpp b/clang/test/CXX/drs/dr20xx.cpp index 4f81b0b413d4bd7aebc16d08a1794112c24b7e15..60ee7684440f54810be19e7010538a37e80a3892 100644 --- a/clang/test/CXX/drs/dr20xx.cpp +++ b/clang/test/CXX/drs/dr20xx.cpp @@ -1,13 +1,14 @@ -// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors \ -// RUN: -Wno-variadic-macros -Wno-c11-extensions -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors - -#if __cplusplus < 201103L -#define static_assert(...) _Static_assert(__VA_ARGS__) +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx14,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx14,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx14,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors + +#if __cplusplus == 199711L +#define static_assert(...) __extension__ _Static_assert(__VA_ARGS__) +// cxx98-error@-1 {{variadic macros are a C99 feature}} #endif namespace dr2007 { // dr2007: 3.4 @@ -15,8 +16,12 @@ template struct A { typename T::error e; }; template struct B { }; B > b1; B > b2 = b1; -int a = b2[0]; // expected-error {{does not provide a subscript operator}} -int b = __builtin_addressof(b2)->foo; // expected-error {{no member}} +int a = b2[0]; +// cxx98-error@-1 {{type 'B >' does not provide a subscript operator}} +// since-cxx11-error@-2 {{type 'B>' does not provide a subscript operator}} +int b = __builtin_addressof(b2)->foo; +// cxx98-error@-1 {{no member named 'foo' in 'dr2007::B >'}} +// since-cxx11-error@-2 {{no member named 'foo' in 'dr2007::B>'}} } // dr2009: na @@ -24,45 +29,69 @@ int b = __builtin_addressof(b2)->foo; // expected-error {{no member}} namespace dr2026 { // dr2026: 11 template struct X {}; - const int a = a + 1; // expected-warning {{uninitialized}} expected-note {{here}} expected-note 0-1{{outside its lifetime}} - X xa; // expected-error {{constant expression}} expected-note {{initializer of 'a'}} + const int a = a + 1; // #dr2026-a + // expected-warning@-1 {{variable 'a' is uninitialized when used within its own initialization}} + X xa; // #dr2026-xa + // cxx98-error@-1 {{non-type template argument of type 'int' is not an integral constant expression}} + // cxx98-note@-2 {{initializer of 'a' is not a constant expression}} + // cxx98-note@#dr2026-a {{declared here}} + // since-cxx11-error@#dr2026-xa {{non-type template argument is not a constant expression}} + // since-cxx11-note@#dr2026-xa {{initializer of 'a' is not a constant expression}} + // since-cxx11-note@#dr2026-a {{declared here}} #if __cplusplus >= 201103L - constexpr int b = b; // expected-error {{constant expression}} expected-note {{outside its lifetime}} - [[clang::require_constant_initialization]] int c = c; // expected-error {{constant initializer}} expected-note {{attribute}} -#if __cplusplus == 201103L - // expected-note@-2 {{read of non-const variable}} expected-note@-2 {{declared here}} -#else - // expected-note@-4 {{outside its lifetime}} -#endif + constexpr int b = b; + // since-cxx11-error@-1 {{constexpr variable 'b' must be initialized by a constant expression}} + // since-cxx11-note@-2 {{read of object outside its lifetime is not allowed in a constant expression}} + [[clang::require_constant_initialization]] int c = c; + // since-cxx11-error@-1 {{variable does not have a constant initializer}} + // since-cxx11-note@-2 {{required by 'require_constant_initialization' attribute here}} + // cxx11-note@-3 {{read of non-const variable 'c' is not allowed in a constant expression}} + // cxx11-note@-4 {{declared here}} + // since-cxx14-note@-5 {{read of object outside its lifetime is not allowed in a constant expression}} #endif -#if __cplusplus > 201703L - constinit int d = d; // expected-error {{constant initializer}} expected-note {{outside its lifetime}} expected-note {{'constinit'}} +#if __cplusplus >= 202002L + constinit int d = d; + // since-cxx20-error@-1 {{variable does not have a constant initializer}} + // since-cxx20-note@-2 {{required by 'constinit' specifier here}} + // since-cxx20-note@-3 {{read of object outside its lifetime is not allowed in a constant expression}} #endif void f() { - static const int e = e + 1; // expected-warning {{suspicious}} expected-note {{here}} expected-note 0-1{{outside its lifetime}} - X xe; // expected-error {{constant expression}} expected-note {{initializer of 'e'}} + static const int e = e + 1; // #dr2026-e + // expected-warning@-1 {{static variable 'e' is suspiciously used within its own initialization}} + X xe; // #dr2026-xe + // cxx98-error@-1 {{non-type template argument of type 'int' is not an integral constant expression}} + // cxx98-note@-2 {{initializer of 'e' is not a constant expression}} + // cxx98-note@#dr2026-e {{declared here}} + // since-cxx11-error@#dr2026-xe {{non-type template argument is not a constant expression}} + // since-cxx11-note@#dr2026-xe {{initializer of 'e' is not a constant expression}} + // since-cxx11-note@#dr2026-e {{declared here}} #if __cplusplus >= 201103L - static constexpr int f = f; // expected-error {{constant expression}} expected-note {{outside its lifetime}} - [[clang::require_constant_initialization]] static int g = g; // expected-error {{constant initializer}} expected-note {{attribute}} -#if __cplusplus == 201103L - // expected-note@-2 {{read of non-const variable}} expected-note@-2 {{declared here}} -#else - // expected-note@-4 {{outside its lifetime}} -#endif + static constexpr int f = f; + // since-cxx11-error@-1 {{constexpr variable 'f' must be initialized by a constant expression}} + // since-cxx11-note@-2 {{read of object outside its lifetime is not allowed in a constant expression}} + [[clang::require_constant_initialization]] static int g = g; + // since-cxx11-error@-1 {{variable does not have a constant initializer}} + // since-cxx11-note@-2 {{required by 'require_constant_initialization' attribute here}} + // cxx11-note@-3 {{read of non-const variable 'g' is not allowed in a constant expression}} + // cxx11-note@-4 {{declared here}} + // since-cxx14-note@-5 {{read of object outside its lifetime is not allowed in a constant expression}} #endif -#if __cplusplus > 201703L - static constinit int h = h; // expected-error {{constant initializer}} expected-note {{outside its lifetime}} expected-note {{'constinit'}} +#if __cplusplus >= 202002L + static constinit int h = h; + // since-cxx20-error@-1 {{variable does not have a constant initializer}} + // since-cxx20-note@-2 {{required by 'constinit' specifier here}} + // since-cxx20-note@-3 {{read of object outside its lifetime is not allowed in a constant expression}} #endif } } namespace dr2049 { // dr2049: 18 drafting -#if __cplusplus > 202002L +#if __cplusplus >= 202302L template struct X {}; X<> a; X b; @@ -120,8 +149,8 @@ namespace dr2076 { // dr2076: 13 operator string_view() const; }; - void foo(const string &); // expected-note {{cannot convert initializer list}} - void bar(string_view); // expected-note 2{{cannot convert initializer list}} + void foo(const string &); // #dr2076-foo + void bar(string_view); // #dr2076-bar void func(const string &arg) { // An argument in one set of braces is subject to user-defined conversions; @@ -130,11 +159,17 @@ namespace dr2076 { // dr2076: 13 foo(arg); foo({arg}); foo({{arg}}); - foo({{{arg}}}); // expected-error {{no matching function}} + foo({{{arg}}}); + // since-cxx11-error@-1 {{no matching function}} + // since-cxx11-note@#dr2076-foo {{cannot convert initializer list}} bar(arg); bar({arg}); - bar({{arg}}); // expected-error {{no matching function}} - bar({{{arg}}}); // expected-error {{no matching function}} + bar({{arg}}); + // since-cxx11-error@-1 {{no matching function}} + // since-cxx11-note@#dr2076-bar {{cannot convert initializer list}} + bar({{{arg}}}); + // since-cxx11-error@-1 {{no matching function}} + // since-cxx11-note@#dr2076-bar {{cannot convert initializer list}} } #endif } @@ -172,18 +207,20 @@ namespace dr2083 { // dr2083: partial // treatment in C++11 onwards. We continue to apply that even after DR2083. void ref_to_non_const() { int c; - const int &ra = a; // expected-note 0-1{{here}} - int &rb = b; // expected-note 0-1{{here}} - int &rc = c; // expected-note {{here}} + const int &ra = a; // #dr2083-ra + int &rb = b; // #dr2083-rb + int &rc = c; // #dr2083-rc struct A { int f() { int a = ra; + // cxx98-error@-1 {{reference to local variable 'ra' declared in enclosing function 'dr2083::ref_to_non_const'}} + // cxx98-note@#dr2083-ra {{'ra' declared here}} int b = rb; -#if __cplusplus < 201103L - // expected-error@-3 {{in enclosing function}} - // expected-error@-3 {{in enclosing function}} -#endif - int c = rc; // expected-error {{in enclosing function}} + // cxx98-error@-1 {{reference to local variable 'rb' declared in enclosing function 'dr2083::ref_to_non_const'}} + // cxx98-note@#dr2083-rb {{'rb' declared here}} + int c = rc; + // expected-error@-1 {{reference to local variable 'rc' declared in enclosing function 'dr2083::ref_to_non_const'}} + // expected-note@#dr2083-rc {{'rc' declared here}} return a + b + c; } }; @@ -207,18 +244,24 @@ namespace dr2083 { // dr2083: partial constexpr NoMut1 nm1 = {1, 2}; constexpr NoMut2 nm2 = {1, 2}; constexpr NoMut3 nm3 = {1, 2}; - constexpr Mut1 m1 = {1, 2}; // expected-note {{declared here}} - constexpr Mut2 m2 = {1, 2}; // expected-note {{declared here}} - constexpr Mut3 m3 = {1, 2}; // expected-note {{declared here}} + constexpr Mut1 m1 = {1, 2}; // #dr2083-m1 + constexpr Mut2 m2 = {1, 2}; // #dr2083-m2 + constexpr Mut3 m3 = {1, 2}; // #dr2083-m3 struct A { void f() { static_assert(nm1.a == 1, ""); static_assert(nm2.m.a == 1, ""); static_assert(nm3.a == 1, ""); // Can't even access a non-mutable member of a variable containing mutable fields. - static_assert(m1.a == 1, ""); // expected-error {{enclosing function}} - static_assert(m2.m.a == 1, ""); // expected-error {{enclosing function}} - static_assert(m3.a == 1, ""); // expected-error {{enclosing function}} + static_assert(m1.a == 1, ""); + // since-cxx11-error@-1 {{reference to local variable 'm1' declared in enclosing function 'dr2083::mutable_subobjects'}} + // since-cxx11-note@#dr2083-m1 {{'m1' declared here}} + static_assert(m2.m.a == 1, ""); + // since-cxx11-error@-1 {{reference to local variable 'm2' declared in enclosing function 'dr2083::mutable_subobjects'}} + // since-cxx11-note@#dr2083-m2 {{'m2' declared here}} + static_assert(m3.a == 1, ""); + // since-cxx11-error@-1 {{reference to local variable 'm3' declared in enclosing function 'dr2083::mutable_subobjects'}} + // since-cxx11-note@#dr2083-m3 {{'m3' declared here}} } }; } @@ -231,14 +274,16 @@ namespace dr2083 { // dr2083: partial #if __cplusplus >= 201103L constexpr #endif - A a = {}; // expected-note {{here}} + A a = {}; // #dr2083-a struct B { void f() { ellipsis(n); // Even though this is technically modelled as an lvalue-to-rvalue // conversion, it calls a constructor and binds 'a' to a reference, so // it results in an odr-use. - ellipsis(a); // expected-error {{enclosing function}} + ellipsis(a); + // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr2083::ellipsis'}} + // expected-note@#dr2083-a {{'a' declared here}} } }; } @@ -246,7 +291,7 @@ namespace dr2083 { // dr2083: partial #if __cplusplus >= 201103L void volatile_lval() { struct A { int n; }; - constexpr A a = {0}; // expected-note {{here}} + constexpr A a = {0}; // #dr2083-a2 struct B { void f() { // An lvalue-to-rvalue conversion of a volatile lvalue always results @@ -254,7 +299,9 @@ namespace dr2083 { // dr2083: partial int A::*p = &A::n; int x = a.*p; volatile int A::*q = p; - int y = a.*q; // expected-error {{enclosing function}} + int y = a.*q; + // since-cxx11-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr2083::volatile_lval'}} + // since-cxx11-note@#dr2083-a2 {{'a' declared here}} } }; } @@ -262,32 +309,45 @@ namespace dr2083 { // dr2083: partial void discarded_lval() { struct A { int x; mutable int y; volatile int z; }; - A a; // expected-note 1+{{here}} - int &r = a.x; // expected-note {{here}} + A a; // #dr2083-a-3 + int &r = a.x; // #dr2083-r struct B { void f() { - a.x; // expected-warning {{unused}} - a.*&A::x; // expected-warning {{unused}} - true ? a.x : a.y; // expected-warning {{unused}} + // FIXME: We emit more errors than we should be. They are explictly marked below. + a.x; + // expected-warning@-1 {{expression result unused}} + // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME + // expected-note@#dr2083-a-3 {{'a' declared here}} + a.*&A::x; + // expected-warning@-1 {{expression result unused}} + // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME + // expected-note@#dr2083-a-3 {{'a' declared here}} + true ? a.x : a.y; // #dr2083-ternary + // expected-warning@-1 {{expression result unused}} + // expected-error@#dr2083-ternary {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME + // expected-note@#dr2083-a-3 {{'a' declared here}} + // expected-error@#dr2083-ternary {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME + // expected-note@#dr2083-a-3 {{'a' declared here}} (void)a.x; - a.x, discarded_lval(); // expected-warning {{left operand of comma operator has no effect}} -#if 1 // FIXME: These errors are all incorrect; the above code is valid. - // expected-error@-6 {{enclosing function}} - // expected-error@-6 {{enclosing function}} - // expected-error@-6 2{{enclosing function}} - // expected-error@-6 {{enclosing function}} - // expected-error@-6 {{enclosing function}} -#endif + // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME + // expected-note@#dr2083-a-3 {{'a' declared here}} + a.x, discarded_lval(); + // expected-warning@-1 {{left operand of comma operator has no effect}} + // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME + // expected-note@#dr2083-a-3 {{'a' declared here}} // 'volatile' qualifier triggers an lvalue-to-rvalue conversion. - a.z; // expected-error {{enclosing function}} -#if __cplusplus < 201103L - // expected-warning@-2 {{assign into a variable}} -#endif + a.z; + // cxx98-warning@-1 {{expression result unused; assign into a variable to force a volatile load}} + // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} + // expected-note@#dr2083-a-3 {{'a' declared here}} // References always get "loaded" to determine what they reference, // even if the result is discarded. - r; // expected-error {{enclosing function}} expected-warning {{unused}} + r; + // expected-warning@-1 {{expression result unused}} + // expected-error@-2 {{reference to local variable 'r' declared in enclosing function 'dr2083::discarded_lval'}} + // expected-note@#dr2083-r {{'r' declared here}} } }; } @@ -295,12 +355,11 @@ namespace dr2083 { // dr2083: partial namespace dr_example_1 { extern int globx; int main() { - const int &x = globx; + const int &x = globx; // #dr2083-x struct A { -#if __cplusplus < 201103L - // expected-error@+2 {{enclosing function}} expected-note@-3 {{here}} -#endif const int *foo() { return &x; } + // cxx98-error@-1 {{reference to local variable 'x' declared in enclosing function 'dr2083::dr_example_1::main'}} + // cxx98-note@#dr2083-x {{'x' declared here}} } a; return *a.foo(); } diff --git a/clang/test/CXX/drs/dr21xx.cpp b/clang/test/CXX/drs/dr21xx.cpp index a1b8fe3f2a9be953cfbf9c1d7b92f284d474c108..a7e50df3f374be9a3a9915212e50131c60060ef1 100644 --- a/clang/test/CXX/drs/dr21xx.cpp +++ b/clang/test/CXX/drs/dr21xx.cpp @@ -1,13 +1,14 @@ -// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors - -#if __cplusplus < 201103L -// expected-error@+1 {{variadic macro}} +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected,cxx98-14,cxx98 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,cxx98-14,since-cxx11 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,cxx98-14,since-cxx11 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors + +#if __cplusplus == 199711L #define static_assert(...) __extension__ _Static_assert(__VA_ARGS__) +// cxx98-error@-1 {{variadic macros are a C99 feature}} #endif namespace dr2100 { // dr2100: 12 @@ -18,15 +19,14 @@ namespace dr2100 { // dr2100: 12 return X<&n>::n; // ok, value-dependent } int g() { - static const int n = 2; + static const int n = 2; // #dr2100-n return X<&n>::n; // ok, value-dependent -#if __cplusplus < 201702L - // expected-error@-2 {{does not have linkage}} expected-note@-3 {{here}} -#endif + // cxx98-14-error@-1 {{non-type template argument refers to object 'n' that does not have linkage}} + // cxx98-14-note@#dr2100-n {{non-type template argument refers to object here}} } }; template struct X