diff options
Diffstat (limited to 'llvm')
158 files changed, 7912 insertions, 5414 deletions
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt index f5f7d3f..651f178 100644 --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -718,6 +718,8 @@ if(LLVM_INDIVIDUAL_TEST_COVERAGE) endif() set(LLVM_LIT_ARGS "${LIT_ARGS_DEFAULT}" CACHE STRING "Default options for lit") +option(LLVM_PARALLEL_LIT "Enable multiple lit suites to run in parallel" OFF) + # On Win32 hosts, provide an option to specify the path to the GnuWin32 tools. if( WIN32 AND NOT CYGWIN ) set(LLVM_LIT_TOOLS_DIR "" CACHE PATH "Path to GnuWin32 tools") diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake index 3bc78b0..0f1734a 100644 --- a/llvm/cmake/modules/AddLLVM.cmake +++ b/llvm/cmake/modules/AddLLVM.cmake @@ -1947,11 +1947,18 @@ function(add_lit_target target comment) list(APPEND LIT_COMMAND --param ${param}) endforeach() if (ARG_UNPARSED_ARGUMENTS) - add_custom_target(${target} - COMMAND ${LIT_COMMAND} ${ARG_UNPARSED_ARGUMENTS} - COMMENT "${comment}" - USES_TERMINAL - ) + if (LLVM_PARALLEL_LIT) + add_custom_target(${target} + COMMAND ${LIT_COMMAND} ${ARG_UNPARSED_ARGUMENTS} + COMMENT "${comment}" + ) + else() + add_custom_target(${target} + COMMAND ${LIT_COMMAND} ${ARG_UNPARSED_ARGUMENTS} + COMMENT "${comment}" + USES_TERMINAL + ) + endif() else() add_custom_target(${target} COMMAND ${CMAKE_COMMAND} -E echo "${target} does nothing, no tools built.") diff --git a/llvm/docs/CMake.rst b/llvm/docs/CMake.rst index abef4f8..35c4798 100644 --- a/llvm/docs/CMake.rst +++ b/llvm/docs/CMake.rst @@ -762,6 +762,12 @@ enabled sub-projects. Nearly all of these variable names begin with **LLVM_PARALLEL_LINK_JOBS**:STRING Define the maximum number of concurrent link jobs. +**LLVM_PARALLEL_LIT**:BOOL + Defaults to ``OFF``. If set to ``OFF``, lit testsuites will be configured + with CMake's ``USES_TERMINAL`` flag to give direct access to the terminal. If + set to ``ON``, that flag will be removed allowing Ninja to schedule multiple + lit testsuites in parallel. + **LLVM_RAM_PER_COMPILE_JOB**:STRING Calculates the amount of Ninja compile jobs according to available resources. Value has to be in MB, overwrites LLVM_PARALLEL_COMPILE_JOBS. Compile jobs diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst index 26ff343..33b0152 100644 --- a/llvm/docs/GlobalISel/GenericOpcode.rst +++ b/llvm/docs/GlobalISel/GenericOpcode.rst @@ -536,15 +536,15 @@ G_FMINIMUM ^^^^^^^^^^ NaN-propagating minimum that also treat -0.0 as less than 0.0. While -FMINNUM_IEEE follow IEEE 754-2008 semantics, FMINIMUM follows IEEE 754-2018 -draft semantics. +FMINNUM_IEEE follow IEEE 754-2008 semantics, FMINIMUM follows IEEE +754-2019 semantics. G_FMAXIMUM ^^^^^^^^^^ NaN-propagating maximum that also treat -0.0 as less than 0.0. While -FMAXNUM_IEEE follow IEEE 754-2008 semantics, FMAXIMUM follows IEEE 754-2018 -draft semantics. +FMAXNUM_IEEE follow IEEE 754-2008 semantics, FMAXIMUM follows IEEE +754-2019 semantics. G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FREM ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 60e682a..f56d4ed 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -15581,7 +15581,7 @@ Semantics: If either operand is a NaN, returns NaN. Otherwise returns the lesser of the two arguments. -0.0 is considered to be less than +0.0 for this intrinsic. Note that these are the semantics specified in the draft of -IEEE 754-2018. +IEEE 754-2019. .. _i_maximum: @@ -15621,7 +15621,7 @@ Semantics: If either operand is a NaN, returns NaN. Otherwise returns the greater of the two arguments. -0.0 is considered to be less than +0.0 for this intrinsic. Note that these are the semantics specified in the draft of -IEEE 754-2018. +IEEE 754-2019. .. _int_copysign: @@ -26000,7 +26000,7 @@ The third argument specifies the exception behavior as described above. Semantics: """""""""" -This function follows semantics specified in the draft of IEEE 754-2018. +This function follows semantics specified in the draft of IEEE 754-2019. '``llvm.experimental.constrained.minimum``' Intrinsic @@ -26032,7 +26032,7 @@ The third argument specifies the exception behavior as described above. Semantics: """""""""" -This function follows semantics specified in the draft of IEEE 754-2018. +This function follows semantics specified in the draft of IEEE 754-2019. '``llvm.experimental.constrained.ceil``' Intrinsic diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst index ed44359..8d293b0 100644 --- a/llvm/docs/RISCVUsage.rst +++ b/llvm/docs/RISCVUsage.rst @@ -117,6 +117,7 @@ on support follow. ``V`` Supported ``Za128rs`` Supported (`See note <#riscv-profiles-extensions-note>`__) ``Za64rs`` Supported (`See note <#riscv-profiles-extensions-note>`__) + ``Zacas`` Supported (`See note <#riscv-zacas-note>`__) ``Zawrs`` Assembly Support ``Zba`` Supported ``Zbb`` Supported @@ -236,6 +237,11 @@ Supported ``Za128rs``, ``Za64rs``, ``Zic64b``, ``Ziccamoa``, ``Ziccif``, ``Zicclsm``, ``Ziccrse``, ``Shcounterenvw``, ``Shgatpa``, ``Shtvala``, ``Shvsatpa``, ``Shvstvala``, ``Shvstvecd``, ``Ssccptr``, ``Sscounterenw``, ``Ssstateen``, ``Ssstrict``, ``Sstvala``, ``Sstvecd``, ``Ssu64xl``, ``Svade``, ``Svbare`` These extensions are defined as part of the `RISC-V Profiles specification <https://github.com/riscv/riscv-profiles/releases/tag/v1.0>`__. They do not introduce any new features themselves, but instead describe existing hardware features. + .. _riscv-zacas-note: + +``Zacas`` + amocas.w will be used for i32 cmpxchg. amocas.d will be used i64 cmpxchg on RV64. The compiler will not generate amocas.d on RV32 or amocas.q on RV64 due to ABI compatibilty. These can only be used in the assembler. + Experimental Extensions ======================= @@ -252,9 +258,6 @@ The primary goal of experimental support is to assist in the process of ratifica ``experimental-zabha`` LLVM implements the `v1.0-rc1 draft specification <https://github.com/riscv/riscv-zabha/tree/v1.0-rc1>`__. -``experimental-zacas`` - LLVM implements the `1.0-rc1 draft specification <https://github.com/riscv/riscv-zacas/releases/tag/v1.0-rc1>`__. - ``experimental-zalasr`` LLVM implements the `0.0.5 draft specification <https://github.com/mehnadnerd/riscv-zalasr>`__. diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst index 51b6527..8ce6ee5 100644 --- a/llvm/docs/ReleaseNotes.rst +++ b/llvm/docs/ReleaseNotes.rst @@ -103,6 +103,7 @@ Changes to the RISC-V Backend * Codegen support was added for the Zimop (May-Be-Operations) extension. * The experimental Ssnpm, Smnpm, Smmpm, Sspm, and Supm 0.8.1 Pointer Masking extensions are supported. * The experimental Ssqosid extension is supported. +* Zacas is no longer experimental. Changes to the WebAssembly Backend ---------------------------------- diff --git a/llvm/include/llvm/ADT/APFloat.h b/llvm/include/llvm/ADT/APFloat.h index 8c247bb..deb74cb 100644 --- a/llvm/include/llvm/ADT/APFloat.h +++ b/llvm/include/llvm/ADT/APFloat.h @@ -1389,29 +1389,35 @@ inline APFloat neg(APFloat X) { return X; } -/// Implements IEEE minNum semantics. Returns the smaller of the 2 arguments if -/// both are not NaN. If either argument is a NaN, returns the other argument. +/// Implements IEEE-754 2019 minimumNumber semantics. Returns the smaller of the +/// 2 arguments if both are not NaN. If either argument is a NaN, returns the +/// other argument. -0 is treated as ordered less than +0. LLVM_READONLY inline APFloat minnum(const APFloat &A, const APFloat &B) { if (A.isNaN()) return B; if (B.isNaN()) return A; + if (A.isZero() && B.isZero() && (A.isNegative() != B.isNegative())) + return A.isNegative() ? A : B; return B < A ? B : A; } -/// Implements IEEE maxNum semantics. Returns the larger of the 2 arguments if -/// both are not NaN. If either argument is a NaN, returns the other argument. +/// Implements IEEE-754 2019 maximumNumber semantics. Returns the larger of the +/// 2 arguments if both are not NaN. If either argument is a NaN, returns the +/// other argument. +0 is treated as ordered greater than -0. LLVM_READONLY inline APFloat maxnum(const APFloat &A, const APFloat &B) { if (A.isNaN()) return B; if (B.isNaN()) return A; + if (A.isZero() && B.isZero() && (A.isNegative() != B.isNegative())) + return A.isNegative() ? B : A; return A < B ? B : A; } -/// Implements IEEE 754-2018 minimum semantics. Returns the smaller of 2 +/// Implements IEEE 754-2019 minimum semantics. Returns the smaller of 2 /// arguments, propagating NaNs and treating -0 as less than +0. LLVM_READONLY inline APFloat minimum(const APFloat &A, const APFloat &B) { @@ -1424,7 +1430,7 @@ inline APFloat minimum(const APFloat &A, const APFloat &B) { return B < A ? B : A; } -/// Implements IEEE 754-2018 maximum semantics. Returns the larger of 2 +/// Implements IEEE 754-2019 maximum semantics. Returns the larger of 2 /// arguments, propagating NaNs and treating -0 as less than +0. LLVM_READONLY inline APFloat maximum(const APFloat &A, const APFloat &B) { diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index 8cb0bc9..ad876c5 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -978,7 +978,7 @@ enum NodeType { /// FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 /// as less than 0.0. While FMINNUM_IEEE/FMAXNUM_IEEE follow IEEE 754-2008 - /// semantics, FMINIMUM/FMAXIMUM follow IEEE 754-2018 draft semantics. + /// semantics, FMINIMUM/FMAXIMUM follow IEEE 754-2019 semantics. FMINIMUM, FMAXIMUM, diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 2fc1cea..25e6c52 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1488,9 +1488,6 @@ public: SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding = false); - SDValue getIndexedStridedLoadVP(SDValue OrigLoad, const SDLoc &DL, - SDValue Base, SDValue Offset, - ISD::MemIndexedMode AM); SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, @@ -1501,9 +1498,6 @@ public: SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT SVT, MachineMemOperand *MMO, bool IsCompressing = false); - SDValue getIndexedStridedStoreVP(SDValue OrigStore, const SDLoc &DL, - SDValue Base, SDValue Offset, - ISD::MemIndexedMode AM); SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef<SDValue> Ops, MachineMemOperand *MMO, diff --git a/llvm/include/llvm/DebugInfo/DIContext.h b/llvm/include/llvm/DebugInfo/DIContext.h index 288ddf7..b75dc8d 100644 --- a/llvm/include/llvm/DebugInfo/DIContext.h +++ b/llvm/include/llvm/DebugInfo/DIContext.h @@ -206,6 +206,7 @@ struct DIDumpOptions { bool IsEH = false; bool DumpNonSkeleton = false; bool ShowAggregateErrors = false; + std::string JsonErrSummaryFile; std::function<llvm::StringRef(uint64_t DwarfRegNum, bool IsEH)> GetNameForDWARFReg; diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h index 4ee51cd..0e81d3b 100644 --- a/llvm/include/llvm/IR/InstrTypes.h +++ b/llvm/include/llvm/IR/InstrTypes.h @@ -468,9 +468,7 @@ public: static BinaryOperator *CreateNeg(Value *Op, const Twine &Name, BasicBlock::iterator InsertBefore); static BinaryOperator *CreateNeg(Value *Op, const Twine &Name = "", - Instruction *InsertBefore = nullptr); - static BinaryOperator *CreateNeg(Value *Op, const Twine &Name, - BasicBlock *InsertAtEnd); + BasicBlock *InsertAtEnd = nullptr); static BinaryOperator *CreateNSWNeg(Value *Op, const Twine &Name, BasicBlock::iterator InsertBefore); static BinaryOperator *CreateNSWNeg(Value *Op, const Twine &Name = "", @@ -1538,10 +1536,19 @@ public: OperandBundleDef OB, Instruction *InsertPt = nullptr); + /// Create a clone of \p CB with operand bundle \p OB added. + static CallBase *addOperandBundle(CallBase *CB, uint32_t ID, + OperandBundleDef OB, + BasicBlock::iterator InsertPt); + /// Create a clone of \p CB with operand bundle \p ID removed. static CallBase *removeOperandBundle(CallBase *CB, uint32_t ID, Instruction *InsertPt = nullptr); + /// Create a clone of \p CB with operand bundle \p ID removed. + static CallBase *removeOperandBundle(CallBase *CB, uint32_t ID, + BasicBlock::iterator InsertPt); + static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Call || I->getOpcode() == Instruction::Invoke || diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index 19197f5..d2036e4 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -815,7 +815,7 @@ def G_FMAXNUM_IEEE : GenericInstruction { // FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 // as less than 0.0. While FMINNUM_IEEE/FMAXNUM_IEEE follow IEEE 754-2008 -// semantics, FMINIMUM/FMAXIMUM follow IEEE 754-2018 draft semantics. +// semantics, FMINIMUM/FMAXIMUM follow IEEE 754-2019 semantics. def G_FMINIMUM : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td index 17757ca..18db7a8 100644 --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -952,6 +952,37 @@ def redundant_binop_in_equality : GICombineRule< [{ return Helper.matchRedundantBinOpInEquality(*${root}, ${info}); }]), (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>; +// Transform: (X == 0 & Y == 0) -> (X | Y) == 0 +def double_icmp_zero_and_combine: GICombineRule< + (defs root:$root), + (match (G_ICMP $d1, $p, $s1, 0), + (G_ICMP $d2, $p, $s2, 0), + (G_AND $root, $d1, $d2), + [{ return ${p}.getPredicate() == CmpInst::ICMP_EQ && + !MRI.getType(${s1}.getReg()).getScalarType().isPointer() && + (MRI.getType(${s1}.getReg()) == + MRI.getType(${s2}.getReg())); }]), + (apply (G_OR $ordst, $s1, $s2), + (G_ICMP $root, $p, $ordst, 0)) +>; + +// Transform: (X != 0 | Y != 0) -> (X | Y) != 0 +def double_icmp_zero_or_combine: GICombineRule< + (defs root:$root), + (match (G_ICMP $d1, $p, $s1, 0), + (G_ICMP $d2, $p, $s2, 0), + (G_OR $root, $d1, $d2), + [{ return ${p}.getPredicate() == CmpInst::ICMP_NE && + !MRI.getType(${s1}.getReg()).getScalarType().isPointer() && + (MRI.getType(${s1}.getReg()) == + MRI.getType(${s2}.getReg())); }]), + (apply (G_OR $ordst, $s1, $s2), + (G_ICMP $root, $p, $ordst, 0)) +>; + +def double_icmp_zero_and_or_combine : GICombineGroup<[double_icmp_zero_and_combine, + double_icmp_zero_or_combine]>; + def and_or_disjoint_mask : GICombineRule< (defs root:$root, build_fn_matchinfo:$info), (match (wip_match_opcode G_AND):$root, @@ -1343,7 +1374,7 @@ def all_combines : GICombineGroup<[trivial_combines, insert_vec_elt_combines, and_or_disjoint_mask, fma_combines, fold_binop_into_select, sub_add_reg, select_to_minmax, redundant_binop_in_equality, fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors, - combine_concat_vector]>; + combine_concat_vector, double_icmp_zero_and_or_combine]>; // A combine group used to for prelegalizer combiners at -O0. The combines in // this group have been selected based on experiments to balance code size and diff --git a/llvm/include/llvm/TextAPI/RecordsSlice.h b/llvm/include/llvm/TextAPI/RecordsSlice.h index 5b214d0..57b23e5 100644 --- a/llvm/include/llvm/TextAPI/RecordsSlice.h +++ b/llvm/include/llvm/TextAPI/RecordsSlice.h @@ -50,9 +50,9 @@ public: /// Add non-ObjC global record. /// /// \param Name The name of symbol. - /// \param Flags The flags that describe attributes of the symbol. - /// \param GV The kind of global. /// \param Linkage The linkage of symbol. + /// \param GV The kind of global. + /// \param Flags The flags that describe attributes of the symbol. /// \return The non-owning pointer to added record in slice. GlobalRecord *addGlobal(StringRef Name, RecordLinkage Linkage, GlobalRecord::Kind GV, @@ -69,6 +69,7 @@ public: /// Add ObjC IVar record. /// + /// \param Container Owning pointer for instance variable. /// \param Name The name of ivar, not symbol. /// \param Linkage The linkage of symbol. /// \return The non-owning pointer to added record in slice. @@ -93,7 +94,7 @@ public: /// Find ObjC Category. /// /// \param ClassToExtend The name of class, not full symbol name. - /// \param Categories The name of category. + /// \param Category The name of category. /// \return The non-owning pointer to record in slice. ObjCCategoryRecord *findObjCCategory(StringRef ClassToExtend, StringRef Category) const; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index e150f27..5b1b7c7c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -3645,32 +3645,42 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, } } } - } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { - // If this is a ZEXTLoad and we are looking at the loaded value. - EVT VT = LD->getMemoryVT(); - unsigned MemBits = VT.getScalarSizeInBits(); - Known.Zero.setBitsFrom(MemBits); - } else if (const MDNode *Ranges = LD->getRanges()) { - EVT VT = LD->getValueType(0); - - // TODO: Handle for extending loads - if (LD->getExtensionType() == ISD::NON_EXTLOAD) { + } else if (Op.getResNo() == 0) { + KnownBits Known0(!LD->getMemoryVT().isScalableVT() + ? LD->getMemoryVT().getFixedSizeInBits() + : BitWidth); + EVT VT = Op.getValueType(); + // Fill in any known bits from range information. There are 3 types being + // used. The results VT (same vector elt size as BitWidth), the loaded + // MemoryVT (which may or may not be vector) and the range VTs original + // type. The range matadata needs the full range (i.e + // MemoryVT().getSizeInBits()), which is truncated to the correct elt size + // if it is know. These are then extended to the original VT sizes below. + if (const MDNode *MD = LD->getRanges()) { + computeKnownBitsFromRangeMetadata(*MD, Known0); if (VT.isVector()) { // Handle truncation to the first demanded element. // TODO: Figure out which demanded elements are covered if (DemandedElts != 1 || !getDataLayout().isLittleEndian()) break; + Known0 = Known0.trunc(BitWidth); + } + } - // Handle the case where a load has a vector type, but scalar memory - // with an attached range. - EVT MemVT = LD->getMemoryVT(); - KnownBits KnownFull(MemVT.getSizeInBits()); + if (LD->getMemoryVT().isVector()) + Known0 = Known0.trunc(LD->getMemoryVT().getScalarSizeInBits()); - computeKnownBitsFromRangeMetadata(*Ranges, KnownFull); - Known = KnownFull.trunc(BitWidth); - } else - computeKnownBitsFromRangeMetadata(*Ranges, Known); - } + // Extend the Known bits from memory to the size of the result. + if (ISD::isZEXTLoad(Op.getNode())) + Known = Known0.zext(BitWidth); + else if (ISD::isSEXTLoad(Op.getNode())) + Known = Known0.sext(BitWidth); + else if (ISD::isEXTLoad(Op.getNode())) + Known = Known0.anyext(BitWidth); + else + Known = Known0; + assert(Known.getBitWidth() == BitWidth); + return Known; } break; } @@ -9106,26 +9116,6 @@ SDValue SelectionDAG::getExtStridedLoadVP( Stride, Mask, EVL, MemVT, MMO, IsExpanding); } -SDValue SelectionDAG::getIndexedStridedLoadVP(SDValue OrigLoad, const SDLoc &DL, - SDValue Base, SDValue Offset, - ISD::MemIndexedMode AM) { - auto *SLD = cast<VPStridedLoadSDNode>(OrigLoad); - assert(SLD->getOffset().isUndef() && - "Strided load is already a indexed load!"); - // Don't propagate the invariant or dereferenceable flags. - auto MMOFlags = - SLD->getMemOperand()->getFlags() & - ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); - MachineFunction &MF = getMachineFunction(); - MachineMemOperand *MMO = MF.getMachineMemOperand( - SLD->getPointerInfo(), MMOFlags, SLD->getMemOperand()->getSize(), - SLD->getOriginalAlign(), SLD->getAAInfo()); - return getStridedLoadVP(AM, SLD->getExtensionType(), OrigLoad.getValueType(), - DL, SLD->getChain(), Base, Offset, SLD->getStride(), - SLD->getMask(), SLD->getVectorLength(), - SLD->getMemoryVT(), MMO, SLD->isExpandingLoad()); -} - SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, @@ -9211,38 +9201,6 @@ SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, return V; } -SDValue SelectionDAG::getIndexedStridedStoreVP(SDValue OrigStore, - const SDLoc &DL, SDValue Base, - SDValue Offset, - ISD::MemIndexedMode AM) { - auto *SST = cast<VPStridedStoreSDNode>(OrigStore); - assert(SST->getOffset().isUndef() && - "Strided store is already an indexed store!"); - SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); - SDValue Ops[] = { - SST->getChain(), SST->getValue(), Base, Offset, SST->getStride(), - SST->getMask(), SST->getVectorLength()}; - FoldingSetNodeID ID; - AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops); - ID.AddInteger(SST->getMemoryVT().getRawBits()); - ID.AddInteger(SST->getRawSubclassData()); - ID.AddInteger(SST->getPointerInfo().getAddrSpace()); - void *IP = nullptr; - if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) - return SDValue(E, 0); - - auto *N = newSDNode<VPStridedStoreSDNode>( - DL.getIROrder(), DL.getDebugLoc(), VTs, AM, SST->isTruncatingStore(), - SST->isCompressingStore(), SST->getMemoryVT(), SST->getMemOperand()); - createOperands(N, Ops); - - CSEMap.InsertNode(N, IP); - InsertNode(N); - SDValue V(N, 0); - NewSDValueDbgMsg(V, "Creating new node: ", this); - return V; -} - SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef<SDValue> Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType) { diff --git a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp index 20ef59e..520debe 100644 --- a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp +++ b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp @@ -29,7 +29,9 @@ #include "llvm/Support/DJB.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FileSystem.h" #include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/JSON.h" #include "llvm/Support/WithColor.h" #include "llvm/Support/raw_ostream.h" #include <map> @@ -2026,12 +2028,37 @@ void OutputCategoryAggregator::EnumerateResults( } void DWARFVerifier::summarize() { - if (ErrorCategory.GetNumCategories() && DumpOpts.ShowAggregateErrors) { + if (DumpOpts.ShowAggregateErrors && ErrorCategory.GetNumCategories()) { error() << "Aggregated error counts:\n"; ErrorCategory.EnumerateResults([&](StringRef s, unsigned count) { error() << s << " occurred " << count << " time(s).\n"; }); } + if (!DumpOpts.JsonErrSummaryFile.empty()) { + std::error_code EC; + raw_fd_ostream JsonStream(DumpOpts.JsonErrSummaryFile, EC, + sys::fs::OF_Text); + if (EC) { + error() << "unable to open json summary file '" + << DumpOpts.JsonErrSummaryFile + << "' for writing: " << EC.message() << '\n'; + return; + } + + llvm::json::Object Categories; + uint64_t ErrorCount = 0; + ErrorCategory.EnumerateResults([&](StringRef Category, unsigned Count) { + llvm::json::Object Val; + Val.try_emplace("count", Count); + Categories.try_emplace(Category, std::move(Val)); + ErrorCount += Count; + }); + llvm::json::Object RootNode; + RootNode.try_emplace("error-categories", std::move(Categories)); + RootNode.try_emplace("error-count", ErrorCount); + + JsonStream << llvm::json::Value(std::move(RootNode)); + } } raw_ostream &DWARFVerifier::error() const { return WithColor::error(OS); } diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index e044ab3..1f3ff22 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -99,8 +99,8 @@ static void findDbgIntrinsics(SmallVectorImpl<IntrinsicT *> &Result, Value *V, SmallPtrSet<DPValue *, 4> EncounteredDPValues; /// Append IntrinsicT users of MetadataAsValue(MD). - auto AppendUsers = [&Ctx, &EncounteredIntrinsics, &Result, - DPValues](Metadata *MD) { + auto AppendUsers = [&Ctx, &EncounteredIntrinsics, &EncounteredDPValues, + &Result, DPValues](Metadata *MD) { if (auto *MDV = MetadataAsValue::getIfExists(Ctx, MD)) { for (User *U : MDV->users()) if (IntrinsicT *DVI = dyn_cast<IntrinsicT>(U)) @@ -113,7 +113,8 @@ static void findDbgIntrinsics(SmallVectorImpl<IntrinsicT *> &Result, Value *V, if (LocalAsMetadata *L = dyn_cast<LocalAsMetadata>(MD)) { for (DPValue *DPV : L->getAllDPValueUsers()) { if (Type == DPValue::LocationType::Any || DPV->getType() == Type) - DPValues->push_back(DPV); + if (EncounteredDPValues.insert(DPV).second) + DPValues->push_back(DPV); } } }; diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp index c54f8d7..ce22175 100644 --- a/llvm/lib/IR/Instruction.cpp +++ b/llvm/lib/IR/Instruction.cpp @@ -46,11 +46,11 @@ Instruction::Instruction(Type *ty, unsigned it, Use *Ops, unsigned NumOps, Instruction::Instruction(Type *ty, unsigned it, Use *Ops, unsigned NumOps, BasicBlock *InsertAtEnd) - : User(ty, Value::InstructionVal + it, Ops, NumOps), Parent(nullptr) { + : User(ty, Value::InstructionVal + it, Ops, NumOps), Parent(nullptr) { - // append this instruction into the basic block - assert(InsertAtEnd && "Basic block to append to may not be NULL!"); - insertInto(InsertAtEnd, InsertAtEnd->end()); + // If requested, append this instruction into the basic block. + if (InsertAtEnd) + insertInto(InsertAtEnd, InsertAtEnd->end()); } Instruction::~Instruction() { @@ -73,7 +73,6 @@ Instruction::~Instruction() { setMetadata(LLVMContext::MD_DIAssignID, nullptr); } - void Instruction::setParent(BasicBlock *P) { Parent = P; } diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index 2577857..42cdcad 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -304,6 +304,20 @@ void LandingPadInst::addClause(Constant *Val) { //===----------------------------------------------------------------------===// CallBase *CallBase::Create(CallBase *CB, ArrayRef<OperandBundleDef> Bundles, + BasicBlock::iterator InsertPt) { + switch (CB->getOpcode()) { + case Instruction::Call: + return CallInst::Create(cast<CallInst>(CB), Bundles, InsertPt); + case Instruction::Invoke: + return InvokeInst::Create(cast<InvokeInst>(CB), Bundles, InsertPt); + case Instruction::CallBr: + return CallBrInst::Create(cast<CallBrInst>(CB), Bundles, InsertPt); + default: + llvm_unreachable("Unknown CallBase sub-class!"); + } +} + +CallBase *CallBase::Create(CallBase *CB, ArrayRef<OperandBundleDef> Bundles, Instruction *InsertPt) { switch (CB->getOpcode()) { case Instruction::Call: @@ -559,6 +573,18 @@ CallBase::BundleOpInfo &CallBase::getBundleOpInfoForOperand(unsigned OpIdx) { CallBase *CallBase::addOperandBundle(CallBase *CB, uint32_t ID, OperandBundleDef OB, + BasicBlock::iterator InsertPt) { + if (CB->getOperandBundle(ID)) + return CB; + + SmallVector<OperandBundleDef, 1> Bundles; + CB->getOperandBundlesAsDefs(Bundles); + Bundles.push_back(OB); + return Create(CB, Bundles, InsertPt); +} + +CallBase *CallBase::addOperandBundle(CallBase *CB, uint32_t ID, + OperandBundleDef OB, Instruction *InsertPt) { if (CB->getOperandBundle(ID)) return CB; @@ -570,6 +596,23 @@ CallBase *CallBase::addOperandBundle(CallBase *CB, uint32_t ID, } CallBase *CallBase::removeOperandBundle(CallBase *CB, uint32_t ID, + BasicBlock::iterator InsertPt) { + SmallVector<OperandBundleDef, 1> Bundles; + bool CreateNew = false; + + for (unsigned I = 0, E = CB->getNumOperandBundles(); I != E; ++I) { + auto Bundle = CB->getOperandBundleAt(I); + if (Bundle.getTagID() == ID) { + CreateNew = true; + continue; + } + Bundles.emplace_back(Bundle); + } + + return CreateNew ? Create(CB, Bundles, InsertPt) : CB; +} + +CallBase *CallBase::removeOperandBundle(CallBase *CB, uint32_t ID, Instruction *InsertPt) { SmallVector<OperandBundleDef, 1> Bundles; bool CreateNew = false; @@ -717,6 +760,13 @@ void CallInst::init(FunctionType *FTy, Value *Func, const Twine &NameStr) { } CallInst::CallInst(FunctionType *Ty, Value *Func, const Twine &Name, + BasicBlock::iterator InsertBefore) + : CallBase(Ty->getReturnType(), Instruction::Call, + OperandTraits<CallBase>::op_end(this) - 1, 1, InsertBefore) { + init(Ty, Func, Name); +} + +CallInst::CallInst(FunctionType *Ty, Value *Func, const Twine &Name, Instruction *InsertBefore) : CallBase(Ty->getReturnType(), Instruction::Call, OperandTraits<CallBase>::op_end(this) - 1, 1, InsertBefore) { @@ -881,6 +931,20 @@ InvokeInst::InvokeInst(const InvokeInst &II) } InvokeInst *InvokeInst::Create(InvokeInst *II, ArrayRef<OperandBundleDef> OpB, + BasicBlock::iterator InsertPt) { + std::vector<Value *> Args(II->arg_begin(), II->arg_end()); + + auto *NewII = InvokeInst::Create( + II->getFunctionType(), II->getCalledOperand(), II->getNormalDest(), + II->getUnwindDest(), Args, OpB, II->getName(), InsertPt); + NewII->setCallingConv(II->getCallingConv()); + NewII->SubclassOptionalData = II->SubclassOptionalData; + NewII->setAttributes(II->getAttributes()); + NewII->setDebugLoc(II->getDebugLoc()); + return NewII; +} + +InvokeInst *InvokeInst::Create(InvokeInst *II, ArrayRef<OperandBundleDef> OpB, Instruction *InsertPt) { std::vector<Value *> Args(II->arg_begin(), II->arg_end()); @@ -954,6 +1018,21 @@ CallBrInst::CallBrInst(const CallBrInst &CBI) } CallBrInst *CallBrInst::Create(CallBrInst *CBI, ArrayRef<OperandBundleDef> OpB, + BasicBlock::iterator InsertPt) { + std::vector<Value *> Args(CBI->arg_begin(), CBI->arg_end()); + + auto *NewCBI = CallBrInst::Create( + CBI->getFunctionType(), CBI->getCalledOperand(), CBI->getDefaultDest(), + CBI->getIndirectDests(), Args, OpB, CBI->getName(), InsertPt); + NewCBI->setCallingConv(CBI->getCallingConv()); + NewCBI->SubclassOptionalData = CBI->SubclassOptionalData; + NewCBI->setAttributes(CBI->getAttributes()); + NewCBI->setDebugLoc(CBI->getDebugLoc()); + NewCBI->NumIndirectDests = CBI->NumIndirectDests; + return NewCBI; +} + +CallBrInst *CallBrInst::Create(CallBrInst *CBI, ArrayRef<OperandBundleDef> OpB, Instruction *InsertPt) { std::vector<Value *> Args(CBI->arg_begin(), CBI->arg_end()); @@ -1138,6 +1217,18 @@ CatchReturnInst::CatchReturnInst(Value *CatchPad, BasicBlock *BB, CatchSwitchInst::CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest, unsigned NumReservedValues, const Twine &NameStr, + BasicBlock::iterator InsertBefore) + : Instruction(ParentPad->getType(), Instruction::CatchSwitch, nullptr, 0, + InsertBefore) { + if (UnwindDest) + ++NumReservedValues; + init(ParentPad, UnwindDest, NumReservedValues + 1); + setName(NameStr); +} + +CatchSwitchInst::CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest, + unsigned NumReservedValues, + const Twine &NameStr, Instruction *InsertBefore) : Instruction(ParentPad->getType(), Instruction::CatchSwitch, nullptr, 0, InsertBefore) { @@ -3224,6 +3315,14 @@ void BinaryOperator::AssertOK() { BinaryOperator *BinaryOperator::Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name, + BasicBlock::iterator InsertBefore) { + assert(S1->getType() == S2->getType() && + "Cannot create binary operator with two operands of differing type!"); + return new BinaryOperator(Op, S1, S2, S1->getType(), Name, InsertBefore); +} + +BinaryOperator *BinaryOperator::Create(BinaryOps Op, Value *S1, Value *S2, + const Twine &Name, Instruction *InsertBefore) { assert(S1->getType() == S2->getType() && "Cannot create binary operator with two operands of differing type!"); @@ -3246,14 +3345,6 @@ BinaryOperator *BinaryOperator::CreateNeg(Value *Op, const Twine &Name, } BinaryOperator *BinaryOperator::CreateNeg(Value *Op, const Twine &Name, - Instruction *InsertBefore) { - Value *Zero = ConstantInt::get(Op->getType(), 0); - return new BinaryOperator(Instruction::Sub, - Zero, Op, - Op->getType(), Name, InsertBefore); -} - -BinaryOperator *BinaryOperator::CreateNeg(Value *Op, const Twine &Name, BasicBlock *InsertAtEnd) { Value *Zero = ConstantInt::get(Op->getType(), 0); return new BinaryOperator(Instruction::Sub, @@ -3286,6 +3377,13 @@ BinaryOperator *BinaryOperator::CreateNUWNeg(Value *Op, const Twine &Name, } BinaryOperator *BinaryOperator::CreateNot(Value *Op, const Twine &Name, + BasicBlock::iterator InsertBefore) { + Constant *C = Constant::getAllOnesValue(Op->getType()); + return new BinaryOperator(Instruction::Xor, Op, C, + Op->getType(), Name, InsertBefore); +} + +BinaryOperator *BinaryOperator::CreateNot(Value *Op, const Twine &Name, Instruction *InsertBefore) { Constant *C = Constant::getAllOnesValue(Op->getType()); return new BinaryOperator(Instruction::Xor, Op, C, @@ -3831,6 +3929,17 @@ CastInst *CastInst::CreatePointerBitCastOrAddrSpaceCast( CastInst *CastInst::CreateBitOrPointerCast(Value *S, Type *Ty, const Twine &Name, + BasicBlock::iterator InsertBefore) { + if (S->getType()->isPointerTy() && Ty->isIntegerTy()) + return Create(Instruction::PtrToInt, S, Ty, Name, InsertBefore); + if (S->getType()->isIntegerTy() && Ty->isPointerTy()) + return Create(Instruction::IntToPtr, S, Ty, Name, InsertBefore); + + return Create(Instruction::BitCast, S, Ty, Name, InsertBefore); +} + +CastInst *CastInst::CreateBitOrPointerCast(Value *S, Type *Ty, + const Twine &Name, Instruction *InsertBefore) { if (S->getType()->isPointerTy() && Ty->isIntegerTy()) return Create(Instruction::PtrToInt, S, Ty, Name, InsertBefore); @@ -4465,6 +4574,18 @@ CmpInst::CmpInst(Type *ty, OtherOps op, Predicate predicate, Value *LHS, CmpInst * CmpInst::Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, + const Twine &Name, BasicBlock::iterator InsertBefore) { + if (Op == Instruction::ICmp) { + return new ICmpInst(InsertBefore, CmpInst::Predicate(predicate), + S1, S2, Name); + } + + return new FCmpInst(InsertBefore, CmpInst::Predicate(predicate), + S1, S2, Name); +} + +CmpInst * +CmpInst::Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, const Twine &Name, Instruction *InsertBefore) { if (Op == Instruction::ICmp) { if (InsertBefore) diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 3741e5d..e0de179 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -5002,7 +5002,9 @@ void Verifier::visitInstruction(Instruction &I) { } else if (GlobalValue *GV = dyn_cast<GlobalValue>(I.getOperand(i))) { Check(GV->getParent() == &M, "Referencing global in another module!", &I, &M, GV, GV->getParent()); - } else if (isa<Instruction>(I.getOperand(i))) { + } else if (Instruction *OpInst = dyn_cast<Instruction>(I.getOperand(i))) { + Check(OpInst->getFunction() == BB->getParent(), + "Referring to an instruction in another function!", &I); verifyDominatesUse(I, i); } else if (isa<InlineAsm>(I.getOperand(i))) { Check(CBI && &CBI->getCalledOperandUse() == &I.getOperandUse(i), diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp index a1c32ee..76a3e50 100644 --- a/llvm/lib/MC/MCParser/AsmParser.cpp +++ b/llvm/lib/MC/MCParser/AsmParser.cpp @@ -44,6 +44,7 @@ #include "llvm/MC/MCSection.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/MC/MCSymbolMachO.h" #include "llvm/MC/MCTargetOptions.h" #include "llvm/MC/MCValue.h" #include "llvm/Support/Casting.h" @@ -1950,7 +1951,8 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info, Lex(); } - if (MAI.hasSubsectionsViaSymbols() && CFIStartProcLoc && Sym->isExternal()) + if (MAI.hasSubsectionsViaSymbols() && CFIStartProcLoc && + Sym->isExternal() && !cast<MCSymbolMachO>(Sym)->isAltEntry()) return Error(StartTokLoc, "non-private labels cannot appear between " ".cfi_startproc / .cfi_endproc pairs") && Error(*CFIStartProcLoc, "previous .cfi_startproc was here"); diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index 142bd50..991c3ac 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -707,7 +707,7 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, // Re-consider control flow based optimizations after redundancy elimination, // redo DCE, etc. - if (EnableDFAJumpThreading && Level.getSizeLevel() == 0) + if (EnableDFAJumpThreading) FPM.addPass(DFAJumpThreadingPass()); FPM.addPass(JumpThreadingPass()); diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index d028302..68f5c36 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -109,6 +109,7 @@ static const RISCVSupportedExtension SupportedExtensions[] = { {"za128rs", {1, 0}}, {"za64rs", {1, 0}}, + {"zacas", {1, 0}}, {"zawrs", {1, 0}}, {"zba", {1, 0}}, @@ -220,7 +221,6 @@ static const RISCVSupportedExtension SupportedExperimentalExtensions[] = { {"zaamo", {0, 2}}, {"zabha", {1, 0}}, - {"zacas", {1, 0}}, {"zalasr", {0, 1}}, {"zalrsc", {0, 2}}, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp index 4f65a95..a0c6bf7 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp @@ -177,7 +177,16 @@ void DivergenceLoweringHelper::buildMergeLaneMasks( B.buildInstr(OrOp, {DstReg}, {PrevMaskedReg, CurMaskedReg}); } -void DivergenceLoweringHelper::constrainAsLaneMask(Incoming &In) { return; } +// GlobalISel has to constrain S1 incoming taken as-is with lane mask register +// class. Insert a copy of Incoming.Reg to new lane mask inside Incoming.Block, +// Incoming.Reg becomes that new lane mask. +void DivergenceLoweringHelper::constrainAsLaneMask(Incoming &In) { + B.setInsertPt(*In.Block, In.Block->getFirstTerminator()); + + auto Copy = B.buildCopy(LLT::scalar(1), In.Reg); + MRI->setRegClass(Copy.getReg(0), ST->getBoolRC()); + In.Reg = Copy.getReg(0); +} } // End anonymous namespace. diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp index aacc359..b2c65e6 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -34,12 +34,6 @@ using namespace llvm; using namespace MIPatternMatch; -static cl::opt<bool> AllowRiskySelect( - "amdgpu-global-isel-risky-select", - cl::desc("Allow GlobalISel to select cases that are likely to not work yet"), - cl::init(false), - cl::ReallyHidden); - #define GET_GLOBALISEL_IMPL #define AMDGPUSubtarget GCNSubtarget #include "AMDGPUGenGlobalISel.inc" @@ -211,14 +205,12 @@ bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const { const Register DefReg = I.getOperand(0).getReg(); const LLT DefTy = MRI->getType(DefReg); - if (DefTy == LLT::scalar(1)) { - if (!AllowRiskySelect) { - LLVM_DEBUG(dbgs() << "Skipping risky boolean phi\n"); - return false; - } - - LLVM_DEBUG(dbgs() << "Selecting risky boolean phi\n"); - } + // S1 G_PHIs should not be selected in instruction-select, instead: + // - divergent S1 G_PHI should go through lane mask merging algorithm + // and be fully inst-selected in AMDGPUGlobalISelDivergenceLowering + // - uniform S1 G_PHI should be lowered into S32 G_PHI in AMDGPURegBankSelect + if (DefTy == LLT::scalar(1)) + return false; // TODO: Verify this doesn't have insane operands (i.e. VGPR to SGPR copy) diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 7f812ed..4b74f3b 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -127,6 +127,7 @@ class MTBUF_Real <MTBUF_Pseudo ps, string real_name = ps.Mnemonic> : // copy relevant pseudo op flags let UseNamedOperandTable = ps.UseNamedOperandTable; let SubtargetPredicate = ps.SubtargetPredicate; + let OtherPredicates = ps.OtherPredicates; let AsmMatchConverter = ps.AsmMatchConverter; let Constraints = ps.Constraints; let DisableEncoding = ps.DisableEncoding; @@ -2508,7 +2509,7 @@ class MUBUF_Real_Atomic_gfx12_impl<bits<8> op, string ps_name, multiclass MUBUF_Real_Atomic_gfx11_Renamed_impl<bits<8> op, bit is_return, string real_name> { - defvar Rtn = !if(!eq(is_return, 1), "_RTN", ""); + defvar Rtn = !if(is_return, "_RTN", ""); def _BOTHEN#Rtn#_gfx11 : MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_BOTHEN" # Rtn, real_name>, AtomicNoRet<NAME # "_BOTHEN_gfx11", is_return>; @@ -2525,7 +2526,7 @@ multiclass MUBUF_Real_Atomic_gfx11_Renamed_impl<bits<8> op, bit is_return, multiclass MUBUF_Real_Atomic_gfx12_Renamed_impl<bits<8> op, bit is_return, string real_name> { - defvar Rtn = !if(!eq(is_return, 1), "_RTN", ""); + defvar Rtn = !if(is_return, "_RTN", ""); def _BOTHEN#Rtn#_gfx12 : MUBUF_Real_Atomic_gfx12_impl<op, NAME # "_VBUFFER_BOTHEN" # Rtn, real_name>, AtomicNoRet<NAME # "_BOTHEN_gfx12", is_return>; diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td index 074e1331..7d79b9b 100644 --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -63,6 +63,7 @@ class DS_Real <DS_Pseudo ps, string opName = ps.Mnemonic> : // copy relevant pseudo op flags let GWS = ps.GWS; let SubtargetPredicate = ps.SubtargetPredicate; + let WaveSizePredicate = ps.WaveSizePredicate; let OtherPredicates = ps.OtherPredicates; let SchedRW = ps.SchedRW; let mayLoad = ps.mayLoad; @@ -1261,7 +1262,9 @@ defm DS_PK_ADD_RTN_BF16 : DS_Real_gfx12<0x0ab>; // New aliases added in GFX12 without renaming the instructions. def : MnemonicAlias<"ds_subrev_u32", "ds_rsub_u32">, Requires<[isGFX12Plus]>; +def : MnemonicAlias<"ds_subrev_rtn_u32", "ds_rsub_rtn_u32">, Requires<[isGFX12Plus]>; def : MnemonicAlias<"ds_subrev_u64", "ds_rsub_u64">, Requires<[isGFX12Plus]>; +def : MnemonicAlias<"ds_subrev_rtn_u64", "ds_rsub_rtn_u64">, Requires<[isGFX12Plus]>; //===----------------------------------------------------------------------===// // GFX11. diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td index a7082f5..8a60168 100644 --- a/llvm/lib/Target/AMDGPU/FLATInstructions.td +++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -153,6 +153,7 @@ class VFLAT_Real <bits<8> op, FLAT_Pseudo ps, string opName = ps.Mnemonic> : // copy relevant pseudo op flags let SubtargetPredicate = ps.SubtargetPredicate; + let WaveSizePredicate = ps.WaveSizePredicate; let AsmMatchConverter = ps.AsmMatchConverter; let OtherPredicates = ps.OtherPredicates; let TSFlags = ps.TSFlags; diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index b2fc7d8..6edf01d 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -856,14 +856,6 @@ bool isReadOnlySegment(const GlobalValue *GV); /// target triple \p TT, false otherwise. bool shouldEmitConstantsToTextSection(const Triple &TT); -/// \returns Integer value requested using \p F's \p Name attribute. -/// -/// \returns \p Default if attribute is not present. -/// -/// \returns \p Default and emits error if requested value cannot be converted -/// to integer. -int getIntegerAttribute(const Function &F, StringRef Name, int Default); - /// \returns A pair of integer values requested using \p F's \p Name attribute /// in "first[,second]" format ("second" is optional unless \p OnlyFirstRequired /// is false). diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td index 918bdb9..fa8d466 100644 --- a/llvm/lib/Target/AMDGPU/VOPInstructions.td +++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td @@ -190,6 +190,7 @@ class VOP3_Real <VOP_Pseudo ps, int EncodingFamily, string asm_name = ps.Mnemoni // copy relevant pseudo op flags let SubtargetPredicate = ps.SubtargetPredicate; + let WaveSizePredicate = ps.WaveSizePredicate; let OtherPredicates = ps.OtherPredicates; let AsmMatchConverter = ps.AsmMatchConverter; let AsmVariantName = ps.AsmVariantName; diff --git a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp index e78ea63..f0b69b0 100644 --- a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp +++ b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp @@ -1468,21 +1468,15 @@ void ARMExpandPseudo::CMSESaveClearFPRegsV8( if (passesFPReg) assert(STI->hasFPRegs() && "Subtarget needs fpregs"); - // Lazy store all fp registers to the stack + // Lazy store all fp registers to the stack. // This executes as NOP in the absence of floating-point support. - MachineInstrBuilder VLSTM = - BuildMI(MBB, MBBI, DL, TII->get(ARM::VLSTM)) - .addReg(ARM::SP) - .add(predOps(ARMCC::AL)) - .addImm(0); // Represents a pseoudo register list, has no effect on - // the encoding. - // Mark non-live registers as undef - for (MachineOperand &MO : VLSTM->implicit_operands()) { - if (MO.isReg() && !MO.isDef()) { - Register Reg = MO.getReg(); - MO.setIsUndef(!LiveRegs.contains(Reg)); - } - } + MachineInstrBuilder VLSTM = BuildMI(MBB, MBBI, DL, TII->get(ARM::VLSTM)) + .addReg(ARM::SP) + .add(predOps(ARMCC::AL)); + for (auto R : {ARM::VPR, ARM::FPSCR, ARM::FPSCR_NZCV, ARM::Q0, ARM::Q1, + ARM::Q2, ARM::Q3, ARM::Q4, ARM::Q5, ARM::Q6, ARM::Q7}) + VLSTM.addReg(R, RegState::Implicit | + (LiveRegs.contains(R) ? 0 : RegState::Undef)); // Restore all arguments for (const auto &Regs : ClearedFPRegs) { @@ -1569,20 +1563,14 @@ void ARMExpandPseudo::CMSESaveClearFPRegsV81(MachineBasicBlock &MBB, .addImm(CMSE_FP_SAVE_SIZE >> 2) .add(predOps(ARMCC::AL)); - // Lazy store all fp registers to the stack. - MachineInstrBuilder VLSTM = - BuildMI(MBB, MBBI, DL, TII->get(ARM::VLSTM)) - .addReg(ARM::SP) - .add(predOps(ARMCC::AL)) - .addImm(0); // Represents a pseoudo register list, has no effect on - // the encoding. - // Mark non-live registers as undef - for (MachineOperand &MO : VLSTM->implicit_operands()) { - if (MO.isReg() && MO.isImplicit() && !MO.isDef()) { - Register Reg = MO.getReg(); - MO.setIsUndef(!LiveRegs.contains(Reg)); - } - } + // Lazy store all FP registers to the stack + MachineInstrBuilder VLSTM = BuildMI(MBB, MBBI, DL, TII->get(ARM::VLSTM)) + .addReg(ARM::SP) + .add(predOps(ARMCC::AL)); + for (auto R : {ARM::VPR, ARM::FPSCR, ARM::FPSCR_NZCV, ARM::Q0, ARM::Q1, + ARM::Q2, ARM::Q3, ARM::Q4, ARM::Q5, ARM::Q6, ARM::Q7}) + VLSTM.addReg(R, RegState::Implicit | + (LiveRegs.contains(R) ? 0 : RegState::Undef)); } else { // Push all the callee-saved registers (s16-s31). MachineInstrBuilder VPUSH = @@ -1685,12 +1673,9 @@ void ARMExpandPseudo::CMSERestoreFPRegsV8( // Lazy load fp regs from stack. // This executes as NOP in the absence of floating-point support. - MachineInstrBuilder VLLDM = - BuildMI(MBB, MBBI, DL, TII->get(ARM::VLLDM)) - .addReg(ARM::SP) - .add(predOps(ARMCC::AL)) - .addImm(0); // Represents a pseoudo register list, has no effect on - // the encoding. + MachineInstrBuilder VLLDM = BuildMI(MBB, MBBI, DL, TII->get(ARM::VLLDM)) + .addReg(ARM::SP) + .add(predOps(ARMCC::AL)); if (STI->fixCMSE_CVE_2021_35465()) { auto Bundler = MIBundleBuilder(MBB, VLLDM); @@ -1772,9 +1757,7 @@ void ARMExpandPseudo::CMSERestoreFPRegsV81( // Load FP registers from stack. BuildMI(MBB, MBBI, DL, TII->get(ARM::VLLDM)) .addReg(ARM::SP) - .add(predOps(ARMCC::AL)) - .addImm(0); // Represents a pseoudo register list, has no effect on the - // encoding. + .add(predOps(ARMCC::AL)); // Pop the stack space BuildMI(MBB, MBBI, DL, TII->get(ARM::tADDspi), ARM::SP) diff --git a/llvm/lib/Target/ARM/ARMInstrFormats.td b/llvm/lib/Target/ARM/ARMInstrFormats.td index 4040858..14e3155 100644 --- a/llvm/lib/Target/ARM/ARMInstrFormats.td +++ b/llvm/lib/Target/ARM/ARMInstrFormats.td @@ -1749,37 +1749,6 @@ class AXSI4<dag oops, dag iops, IndexMode im, InstrItinClass itin, let Inst{8} = 0; // Single precision } -// Single Precision with fixed registers. -// For when the registers-to-be-stored/loaded are fixed, e.g. VLLDM and VLSTM -class AXSI4FR<string asm, bit et, bit load> - : InstARM<AddrMode4, 4, IndexModeNone, VFPLdStMulFrm, VFPDomain, "", NoItinerary> { - // Instruction operands. - bits<4> Rn; - bits<13> regs; // Does not affect encoding, for assembly/disassembly only. - list<Predicate> Predicates = [HasVFP2]; - let OutOperandList = (outs); - let InOperandList = (ins GPRnopc:$Rn, pred:$p, dpr_reglist:$regs); - let AsmString = asm; - let Pattern = []; - let DecoderNamespace = "VFP"; - // Encode instruction operands. - let Inst{19-16} = Rn; - let Inst{31-28} = 0b1110; - let Inst{27-25} = 0b110; - let Inst{24} = 0b0; - let Inst{23} = 0b0; - let Inst{22} = 0b0; - let Inst{21} = 0b1; - let Inst{20} = load; // Distinguishes vlldm from vlstm - let Inst{15-12} = 0b0000; - let Inst{11-9} = 0b101; - let Inst{8} = 0; // Single precision - let Inst{7} = et; // encoding type, 0 for T1 and 1 for T2. - let Inst{6-0} = 0b0000000; - let mayLoad = load; - let mayStore = !eq(load, 0); -} - // Double precision, unary class ADuI<bits<5> opcod1, bits<2> opcod2, bits<4> opcod3, bits<2> opcod4, bit opcod5, dag oops, dag iops, InstrItinClass itin, string opc, diff --git a/llvm/lib/Target/ARM/ARMInstrVFP.td b/llvm/lib/Target/ARM/ARMInstrVFP.td index 3094a4d..55d3efb 100644 --- a/llvm/lib/Target/ARM/ARMInstrVFP.td +++ b/llvm/lib/Target/ARM/ARMInstrVFP.td @@ -313,51 +313,29 @@ def : MnemonicAlias<"vstm", "vstmia">; //===----------------------------------------------------------------------===// // Lazy load / store multiple Instructions // -// VLLDM and VLSTM: -// 2 encoding options: -// T1 (bit 7 is 0): -// T1 takes an optional dpr_reglist, must be '{d0-d15}' (exactly) -// T1 require v8-M.Main, secure state, target with 16 D registers (or with no D registers - NOP) -// T2 (bit 7 is 1): -// T2 takes a mandatory dpr_reglist, must be '{d0-d31}' (exactly) -// T2 require v8.1-M.Main, secure state, target with 16/32 D registers (or with no D registers - NOP) -// (source: Arm v8-M ARM, DDI0553B.v ID16122022) - -def VLLDM : AXSI4FR<"vlldm${p}\t$Rn, $regs", 0, 1>, +def VLLDM : AXSI4<(outs), (ins GPRnopc:$Rn, pred:$p), IndexModeNone, + NoItinerary, "vlldm${p}\t$Rn", "", []>, Requires<[HasV8MMainline, Has8MSecExt]> { - let Defs = [VPR, FPSCR, FPSCR_NZCV, D0, D1, D2, D3, D4, D5, D6, D7, D8, D9, D10, D11, D12, D13, D14, D15]; - let DecoderMethod = "DecodeLazyLoadStoreMul"; -} -// T1: assembly does not contains the register list. -def : InstAlias<"vlldm${p}\t$Rn", (VLLDM GPRnopc:$Rn, pred:$p, 0)>, - Requires<[HasV8MMainline, Has8MSecExt]>; -// T2: assembly must contains the register list. -// The register list has no effect on the encoding, it is for assembly/disassembly purposes only. -def VLLDM_T2 : AXSI4FR<"vlldm${p}\t$Rn, $regs", 1, 1>, - Requires<[HasV8_1MMainline, Has8MSecExt]> { - let Defs = [VPR, FPSCR, FPSCR_NZCV, D0, D1, D2, D3, D4, D5, D6, D7, D8, D9, D10, D11, D12, D13, D14, D15, - D16, D17, D18, D19, D20, D21, D22, D23, D24, D25, D26, D27, D28, D29, D30, D31]; - let DecoderMethod = "DecodeLazyLoadStoreMul"; -} -// T1: assembly contains the register list. -// The register list has no effect on the encoding, it is for assembly/disassembly purposes only. -def VLSTM : AXSI4FR<"vlstm${p}\t$Rn, $regs", 0, 0>, + let Inst{24-23} = 0b00; + let Inst{22} = 0; + let Inst{21} = 1; + let Inst{20} = 1; + let Inst{15-12} = 0; + let Inst{7-0} = 0; + let mayLoad = 1; + let Defs = [Q0, Q1, Q2, Q3, Q4, Q5, Q6, Q7, VPR, FPSCR, FPSCR_NZCV]; +} + +def VLSTM : AXSI4<(outs), (ins GPRnopc:$Rn, pred:$p), IndexModeNone, + NoItinerary, "vlstm${p}\t$Rn", "", []>, Requires<[HasV8MMainline, Has8MSecExt]> { - let Defs = [VPR, FPSCR, FPSCR_NZCV]; - let Uses = [VPR, FPSCR, FPSCR_NZCV, D0, D1, D2, D3, D4, D5, D6, D7, D8, D9, D10, D11, D12, D13, D14, D15]; - let DecoderMethod = "DecodeLazyLoadStoreMul"; -} -// T1: assembly does not contain the register list. -def : InstAlias<"vlstm${p}\t$Rn", (VLSTM GPRnopc:$Rn, pred:$p, 0)>, - Requires<[HasV8MMainline, Has8MSecExt]>; -// T2: assembly must contain the register list. -// The register list has no effect on the encoding, it is for assembly/disassembly purposes only. -def VLSTM_T2 : AXSI4FR<"vlstm${p}\t$Rn, $regs", 1, 0>, - Requires<[HasV8_1MMainline, Has8MSecExt]> { - let Defs = [VPR, FPSCR, FPSCR_NZCV]; - let Uses = [VPR, FPSCR, FPSCR_NZCV, D0, D1, D2, D3, D4, D5, D6, D7, D8, D9, D10, D11, D12, D13, D14, D15, - D16, D17, D18, D19, D20, D21, D22, D23, D24, D25, D26, D27, D28, D29, D30, D31]; - let DecoderMethod = "DecodeLazyLoadStoreMul"; + let Inst{24-23} = 0b00; + let Inst{22} = 0; + let Inst{21} = 1; + let Inst{20} = 0; + let Inst{15-12} = 0; + let Inst{7-0} = 0; + let mayStore = 1; } def : InstAlias<"vpush${p} $r", (VSTMDDB_UPD SP, pred:$p, dpr_reglist:$r), 0>, diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp index 5efbaf0..37bfb76 100644 --- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp +++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp @@ -450,12 +450,11 @@ class ARMAsmParser : public MCTargetAsmParser { bool validatetSTMRegList(const MCInst &Inst, const OperandVector &Operands, unsigned ListNo); - int tryParseRegister(bool AllowOutofBoundReg = false); + int tryParseRegister(); bool tryParseRegisterWithWriteBack(OperandVector &); int tryParseShiftRegister(OperandVector &); bool parseRegisterList(OperandVector &, bool EnforceOrder = true, - bool AllowRAAC = false, - bool AllowOutOfBoundReg = false); + bool AllowRAAC = false); bool parseMemory(OperandVector &); bool parseOperand(OperandVector &, StringRef Mnemonic); bool parseImmExpr(int64_t &Out); @@ -4073,7 +4072,7 @@ ParseStatus ARMAsmParser::tryParseRegister(MCRegister &Reg, SMLoc &StartLoc, /// Try to parse a register name. The token must be an Identifier when called, /// and if it is a register name the token is eaten and the register number is /// returned. Otherwise return -1. -int ARMAsmParser::tryParseRegister(bool AllowOutOfBoundReg) { +int ARMAsmParser::tryParseRegister() { MCAsmParser &Parser = getParser(); const AsmToken &Tok = Parser.getTok(); if (Tok.isNot(AsmToken::Identifier)) return -1; @@ -4117,8 +4116,7 @@ int ARMAsmParser::tryParseRegister(bool AllowOutOfBoundReg) { } // Some FPUs only have 16 D registers, so D16-D31 are invalid - if (!AllowOutOfBoundReg && !hasD32() && RegNum >= ARM::D16 && - RegNum <= ARM::D31) + if (!hasD32() && RegNum >= ARM::D16 && RegNum <= ARM::D31) return -1; Parser.Lex(); // Eat identifier token. @@ -4458,7 +4456,7 @@ insertNoDuplicates(SmallVectorImpl<std::pair<unsigned, unsigned>> &Regs, /// Parse a register list. bool ARMAsmParser::parseRegisterList(OperandVector &Operands, bool EnforceOrder, - bool AllowRAAC, bool AllowOutOfBoundReg) { + bool AllowRAAC) { MCAsmParser &Parser = getParser(); if (Parser.getTok().isNot(AsmToken::LCurly)) return TokError("Token is not a Left Curly Brace"); @@ -4512,7 +4510,7 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands, bool EnforceOrder, return Error(RegLoc, "pseudo-register not allowed"); Parser.Lex(); // Eat the minus. SMLoc AfterMinusLoc = Parser.getTok().getLoc(); - int EndReg = tryParseRegister(AllowOutOfBoundReg); + int EndReg = tryParseRegister(); if (EndReg == -1) return Error(AfterMinusLoc, "register expected"); if (EndReg == ARM::RA_AUTH_CODE) @@ -4547,7 +4545,7 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands, bool EnforceOrder, RegLoc = Parser.getTok().getLoc(); int OldReg = Reg; const AsmToken RegTok = Parser.getTok(); - Reg = tryParseRegister(AllowOutOfBoundReg); + Reg = tryParseRegister(); if (Reg == -1) return Error(RegLoc, "register expected"); if (!AllowRAAC && Reg == ARM::RA_AUTH_CODE) @@ -6087,11 +6085,8 @@ bool ARMAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) { } case AsmToken::LBrac: return parseMemory(Operands); - case AsmToken::LCurly: { - bool AllowOutOfBoundReg = Mnemonic == "vlldm" || Mnemonic == "vlstm"; - return parseRegisterList(Operands, !Mnemonic.starts_with("clr"), false, - AllowOutOfBoundReg); - } + case AsmToken::LCurly: + return parseRegisterList(Operands, !Mnemonic.starts_with("clr")); case AsmToken::Dollar: case AsmToken::Hash: { // #42 -> immediate @@ -7601,33 +7596,6 @@ bool ARMAsmParser::validateInstruction(MCInst &Inst, const unsigned Opcode = Inst.getOpcode(); switch (Opcode) { - case ARM::VLLDM: - case ARM::VLLDM_T2: - case ARM::VLSTM: - case ARM::VLSTM_T2: { - // Since in some cases both T1 and T2 are valid, tablegen can not always - // pick the correct instruction. - if (Operands.size() == 4) { // a register list has been provided - ARMOperand &Op = static_cast<ARMOperand &>( - *Operands[3]); // the register list, a dpr_reglist - assert(Op.isDPRRegList()); - auto &RegList = Op.getRegList(); - // T2 requires v8.1-M.Main (cannot be handled by tablegen) - if (RegList.size() == 32 && !hasV8_1MMainline()) { - return Error(Op.getEndLoc(), "T2 version requires v8.1-M.Main"); - } - // When target has 32 D registers, T1 is undefined. - if (hasD32() && RegList.size() != 32) { - return Error(Op.getEndLoc(), "operand must be exactly {d0-d31}"); - } - // When target has 16 D registers, both T1 and T2 are valid. - if (!hasD32() && (RegList.size() != 16 && RegList.size() != 32)) { - return Error(Op.getEndLoc(), - "operand must be exactly {d0-d15} (T1) or {d0-d31} (T2)"); - } - } - return false; - } case ARM::t2IT: { // Encoding is unpredictable if it ever results in a notional 'NV' // predicate. Since we don't parse 'NV' directly this means an 'AL' @@ -8763,32 +8731,6 @@ bool ARMAsmParser::processInstruction(MCInst &Inst, } switch (Inst.getOpcode()) { - case ARM::VLLDM: - case ARM::VLSTM: { - // In some cases both T1 and T2 are valid, causing tablegen pick T1 instead - // of T2 - if (Operands.size() == 4) { // a register list has been provided - ARMOperand &Op = static_cast<ARMOperand &>( - *Operands[3]); // the register list, a dpr_reglist - assert(Op.isDPRRegList()); - auto &RegList = Op.getRegList(); - // When the register list is {d0-d31} the instruction has to be the T2 - // variant - if (RegList.size() == 32) { - const unsigned Opcode = - (Inst.getOpcode() == ARM::VLLDM) ? ARM::VLLDM_T2 : ARM::VLSTM_T2; - MCInst TmpInst; - TmpInst.setOpcode(Opcode); - TmpInst.addOperand(Inst.getOperand(0)); - TmpInst.addOperand(Inst.getOperand(1)); - TmpInst.addOperand(Inst.getOperand(2)); - TmpInst.addOperand(Inst.getOperand(3)); - Inst = TmpInst; - return true; - } - } - return false; - } // Alias for alternate form of 'ldr{,b}t Rt, [Rn], #imm' instruction. case ARM::LDRT_POST: case ARM::LDRBT_POST: { diff --git a/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp b/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp index 705f3cbc..604f22d 100644 --- a/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp +++ b/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp @@ -700,9 +700,6 @@ DecodeMVEOverlappingLongShift(MCInst &Inst, unsigned Insn, uint64_t Address, static DecodeStatus DecodeT2AddSubSPImm(MCInst &Inst, unsigned Insn, uint64_t Address, const MCDisassembler *Decoder); -static DecodeStatus DecodeLazyLoadStoreMul(MCInst &Inst, unsigned Insn, - uint64_t Address, - const MCDisassembler *Decoder); #include "ARMGenDisassemblerTables.inc" @@ -7033,23 +7030,3 @@ static DecodeStatus DecodeT2AddSubSPImm(MCInst &Inst, unsigned Insn, return DS; } - -static DecodeStatus DecodeLazyLoadStoreMul(MCInst &Inst, unsigned Insn, - uint64_t Address, - const MCDisassembler *Decoder) { - DecodeStatus S = MCDisassembler::Success; - - const unsigned Rn = fieldFromInstruction(Insn, 16, 4); - // Adding Rn, holding memory location to save/load to/from, the only argument - // that is being encoded. - // '$Rn' in the assembly. - if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder))) - return MCDisassembler::Fail; - // An optional predicate, '$p' in the assembly. - DecodePredicateOperand(Inst, ARMCC::AL, Address, Decoder); - // An immediate that represents a floating point registers list. '$regs' in - // the assembly. - Inst.addOperand(MCOperand::createImm(0)); // Arbitrary value, has no effect. - - return S; -} diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp index 24e627c..fbd067d 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp @@ -91,38 +91,6 @@ void ARMInstPrinter::printInst(const MCInst *MI, uint64_t Address, unsigned Opcode = MI->getOpcode(); switch (Opcode) { - case ARM::VLLDM: { - const MCOperand &Reg = MI->getOperand(0); - O << '\t' << "vlldm" << '\t'; - printRegName(O, Reg.getReg()); - O << ", " - << "{d0 - d15}"; - return; - } - case ARM::VLLDM_T2: { - const MCOperand &Reg = MI->getOperand(0); - O << '\t' << "vlldm" << '\t'; - printRegName(O, Reg.getReg()); - O << ", " - << "{d0 - d31}"; - return; - } - case ARM::VLSTM: { - const MCOperand &Reg = MI->getOperand(0); - O << '\t' << "vlstm" << '\t'; - printRegName(O, Reg.getReg()); - O << ", " - << "{d0 - d15}"; - return; - } - case ARM::VLSTM_T2: { - const MCOperand &Reg = MI->getOperand(0); - O << '\t' << "vlstm" << '\t'; - printRegName(O, Reg.getReg()); - O << ", " - << "{d0 - d31}"; - return; - } // Check for MOVs and print canonical forms, instead. case ARM::MOVsr: { // FIXME: Thumb variants? diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index 8a3454c..67ef798 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -12,139 +12,224 @@ //===----------------------------------------------------------------------===// include "llvm/IR/Intrinsics.td" -include "llvm/IR/Attributes.td" -// Abstract representation of the class a DXIL Operation belongs to. -class DXILOpClass<string name> { - string Name = name; +class DXILOpClass; + +// Following is a set of DXIL Operation classes whose names appear to be +// arbitrary, yet need to be a substring of the function name used during +// lowering to DXIL Operation calls. These class name strings are specified +// as the third argument of add_dixil_op in utils/hct/hctdb.py and case converted +// in utils/hct/hctdb_instrhelp.py of DirectXShaderCompiler repo. The function +// name has the format "dx.op.<class-name>.<return-type>". + +defset list<DXILOpClass> OpClasses = { + def acceptHitAndEndSearch : DXILOpClass; + def allocateNodeOutputRecords : DXILOpClass; + def allocateRayQuery : DXILOpClass; + def annotateHandle : DXILOpClass; + def annotateNodeHandle : DXILOpClass; + def annotateNodeRecordHandle : DXILOpClass; + def atomicBinOp : DXILOpClass; + def atomicCompareExchange : DXILOpClass; + def attributeAtVertex : DXILOpClass; + def barrier : DXILOpClass; + def barrierByMemoryHandle : DXILOpClass; + def barrierByMemoryType : DXILOpClass; + def barrierByNodeRecordHandle : DXILOpClass; + def binary : DXILOpClass; + def binaryWithCarryOrBorrow : DXILOpClass; + def binaryWithTwoOuts : DXILOpClass; + def bitcastF16toI16 : DXILOpClass; + def bitcastF32toI32 : DXILOpClass; + def bitcastF64toI64 : DXILOpClass; + def bitcastI16toF16 : DXILOpClass; + def bitcastI32toF32 : DXILOpClass; + def bitcastI64toF64 : DXILOpClass; + def bufferLoad : DXILOpClass; + def bufferStore : DXILOpClass; + def bufferUpdateCounter : DXILOpClass; + def calculateLOD : DXILOpClass; + def callShader : DXILOpClass; + def cbufferLoad : DXILOpClass; + def cbufferLoadLegacy : DXILOpClass; + def checkAccessFullyMapped : DXILOpClass; + def coverage : DXILOpClass; + def createHandle : DXILOpClass; + def createHandleForLib : DXILOpClass; + def createHandleFromBinding : DXILOpClass; + def createHandleFromHeap : DXILOpClass; + def createNodeInputRecordHandle : DXILOpClass; + def createNodeOutputHandle : DXILOpClass; + def cutStream : DXILOpClass; + def cycleCounterLegacy : DXILOpClass; + def discard : DXILOpClass; + def dispatchMesh : DXILOpClass; + def dispatchRaysDimensions : DXILOpClass; + def dispatchRaysIndex : DXILOpClass; + def domainLocation : DXILOpClass; + def dot2 : DXILOpClass; + def dot2AddHalf : DXILOpClass; + def dot3 : DXILOpClass; + def dot4 : DXILOpClass; + def dot4AddPacked : DXILOpClass; + def emitIndices : DXILOpClass; + def emitStream : DXILOpClass; + def emitThenCutStream : DXILOpClass; + def evalCentroid : DXILOpClass; + def evalSampleIndex : DXILOpClass; + def evalSnapped : DXILOpClass; + def finishedCrossGroupSharing : DXILOpClass; + def flattenedThreadIdInGroup : DXILOpClass; + def geometryIndex : DXILOpClass; + def getDimensions : DXILOpClass; + def getInputRecordCount : DXILOpClass; + def getMeshPayload : DXILOpClass; + def getNodeRecordPtr : DXILOpClass; + def getRemainingRecursionLevels : DXILOpClass; + def groupId : DXILOpClass; + def gsInstanceID : DXILOpClass; + def hitKind : DXILOpClass; + def ignoreHit : DXILOpClass; + def incrementOutputCount : DXILOpClass; + def indexNodeHandle : DXILOpClass; + def innerCoverage : DXILOpClass; + def instanceID : DXILOpClass; + def instanceIndex : DXILOpClass; + def isHelperLane : DXILOpClass; + def isSpecialFloat : DXILOpClass; + def legacyDoubleToFloat : DXILOpClass; + def legacyDoubleToSInt32 : DXILOpClass; + def legacyDoubleToUInt32 : DXILOpClass; + def legacyF16ToF32 : DXILOpClass; + def legacyF32ToF16 : DXILOpClass; + def loadInput : DXILOpClass; + def loadOutputControlPoint : DXILOpClass; + def loadPatchConstant : DXILOpClass; + def makeDouble : DXILOpClass; + def minPrecXRegLoad : DXILOpClass; + def minPrecXRegStore : DXILOpClass; + def nodeOutputIsValid : DXILOpClass; + def objectRayDirection : DXILOpClass; + def objectRayOrigin : DXILOpClass; + def objectToWorld : DXILOpClass; + def outputComplete : DXILOpClass; + def outputControlPointID : DXILOpClass; + def pack4x8 : DXILOpClass; + def primitiveID : DXILOpClass; + def primitiveIndex : DXILOpClass; + def quadOp : DXILOpClass; + def quadReadLaneAt : DXILOpClass; + def quadVote : DXILOpClass; + def quaternary : DXILOpClass; + def rawBufferLoad : DXILOpClass; + def rawBufferStore : DXILOpClass; + def rayFlags : DXILOpClass; + def rayQuery_Abort : DXILOpClass; + def rayQuery_CommitNonOpaqueTriangleHit : DXILOpClass; + def rayQuery_CommitProceduralPrimitiveHit : DXILOpClass; + def rayQuery_Proceed : DXILOpClass; + def rayQuery_StateMatrix : DXILOpClass; + def rayQuery_StateScalar : DXILOpClass; + def rayQuery_StateVector : DXILOpClass; + def rayQuery_TraceRayInline : DXILOpClass; + def rayTCurrent : DXILOpClass; + def rayTMin : DXILOpClass; + def renderTargetGetSampleCount : DXILOpClass; + def renderTargetGetSamplePosition : DXILOpClass; + def reportHit : DXILOpClass; + def sample : DXILOpClass; + def sampleBias : DXILOpClass; + def sampleCmp : DXILOpClass; + def sampleCmpBias : DXILOpClass; + def sampleCmpGrad : DXILOpClass; + def sampleCmpLevel : DXILOpClass; + def sampleCmpLevelZero : DXILOpClass; + def sampleGrad : DXILOpClass; + def sampleIndex : DXILOpClass; + def sampleLevel : DXILOpClass; + def setMeshOutputCounts : DXILOpClass; + def splitDouble : DXILOpClass; + def startInstanceLocation : DXILOpClass; + def startVertexLocation : DXILOpClass; + def storeOutput : DXILOpClass; + def storePatchConstant : DXILOpClass; + def storePrimitiveOutput : DXILOpClass; + def storeVertexOutput : DXILOpClass; + def tempRegLoad : DXILOpClass; + def tempRegStore : DXILOpClass; + def tertiary : DXILOpClass; + def texture2DMSGetSamplePosition : DXILOpClass; + def textureGather : DXILOpClass; + def textureGatherCmp : DXILOpClass; + def textureGatherRaw : DXILOpClass; + def textureLoad : DXILOpClass; + def textureStore : DXILOpClass; + def textureStoreSample : DXILOpClass; + def threadId : DXILOpClass; + def threadIdInGroup : DXILOpClass; + def traceRay : DXILOpClass; + def unary : DXILOpClass; + def unaryBits : DXILOpClass; + def unpack4x8 : DXILOpClass; + def viewID : DXILOpClass; + def waveActiveAllEqual : DXILOpClass; + def waveActiveBallot : DXILOpClass; + def waveActiveBit : DXILOpClass; + def waveActiveOp : DXILOpClass; + def waveAllOp : DXILOpClass; + def waveAllTrue : DXILOpClass; + def waveAnyTrue : DXILOpClass; + def waveGetLaneCount : DXILOpClass; + def waveGetLaneIndex : DXILOpClass; + def waveIsFirstLane : DXILOpClass; + def waveMatch : DXILOpClass; + def waveMatrix_Accumulate : DXILOpClass; + def waveMatrix_Annotate : DXILOpClass; + def waveMatrix_Depth : DXILOpClass; + def waveMatrix_Fill : DXILOpClass; + def waveMatrix_LoadGroupShared : DXILOpClass; + def waveMatrix_LoadRawBuf : DXILOpClass; + def waveMatrix_Multiply : DXILOpClass; + def waveMatrix_ScalarOp : DXILOpClass; + def waveMatrix_StoreGroupShared : DXILOpClass; + def waveMatrix_StoreRawBuf : DXILOpClass; + def waveMultiPrefixBitCount : DXILOpClass; + def waveMultiPrefixOp : DXILOpClass; + def wavePrefixOp : DXILOpClass; + def waveReadLaneAt : DXILOpClass; + def waveReadLaneFirst : DXILOpClass; + def worldRayDirection : DXILOpClass; + def worldRayOrigin : DXILOpClass; + def worldToObject : DXILOpClass; + def writeSamplerFeedback : DXILOpClass; + def writeSamplerFeedbackBias : DXILOpClass; + def writeSamplerFeedbackGrad : DXILOpClass; + def writeSamplerFeedbackLevel: DXILOpClass; } -// Abstract representation of the category a DXIL Operation belongs to -class DXILOpCategory<string name> { - string Name = name; +// Abstraction DXIL Operation to LLVM intrinsic +class DXILOpMapping<int opCode, DXILOpClass opClass, Intrinsic intrinsic, string doc> { + int OpCode = opCode; // Opcode corresponding to DXIL Operation + DXILOpClass OpClass = opClass; // Class of DXIL Operation. + Intrinsic LLVMIntrinsic = intrinsic; // LLVM Intrinsic the DXIL Operation maps + string Doc = doc; // to a short description of the operation } -def UnaryClass : DXILOpClass<"Unary">; -def BinaryClass : DXILOpClass<"Binary">; -def FlattenedThreadIdInGroupClass : DXILOpClass<"FlattenedThreadIdInGroup">; -def ThreadIdInGroupClass : DXILOpClass<"ThreadIdInGroup">; -def ThreadIdClass : DXILOpClass<"ThreadId">; -def GroupIdClass : DXILOpClass<"GroupId">; - -def BinaryUintCategory : DXILOpCategory<"Binary uint">; -def UnaryFloatCategory : DXILOpCategory<"Unary float">; -def ComputeIDCategory : DXILOpCategory<"Compute/Mesh/Amplification shader">; - -// Represent as any pointer type with an option to change to a qualified pointer -// type with address space specified. -def dxil_handle_ty : LLVMAnyPointerType; -def dxil_cbuffer_ty : LLVMAnyPointerType; -def dxil_resource_ty : LLVMAnyPointerType; - -// The parameter description for a DXIL operation -class DXILOpParameter<int pos, LLVMType type, string name, string doc, - bit isConstant = 0, string enumName = "", - int maxValue = 0> { - int Pos = pos; // Position in parameter list - LLVMType ParamType = type; // Parameter type - string Name = name; // Short, unique parameter name - string Doc = doc; // Description of this parameter - bit IsConstant = isConstant; // Whether this parameter requires a constant value in the IR - string EnumName = enumName; // Name of the enum type, if applicable - int MaxValue = maxValue; // Maximum value for this parameter, if applicable -} - -// A representation for a DXIL operation -class DXILOperationDesc { - string OpName = ""; // Name of DXIL operation - int OpCode = 0; // Unique non-negative integer associated with the operation - DXILOpClass OpClass; // Class of the operation - DXILOpCategory OpCategory; // Category of the operation - string Doc = ""; // Description of the operation - list<DXILOpParameter> Params = []; // Parameter list of the operation - list<LLVMType> OverloadTypes = []; // Overload types, if applicable - EnumAttr Attribute; // Operation Attribute. Leverage attributes defined in Attributes.td - // ReadNone - operation does not access memory. - // ReadOnly - only reads from memory. - // "ReadMemory" - reads memory - bit IsDerivative = 0; // Whether this is some kind of derivative - bit IsGradient = 0; // Whether this requires a gradient calculation - bit IsFeedback = 0; // Whether this is a sampler feedback operation - bit IsWave = 0; // Whether this requires in-wave, cross-lane functionality - bit NeedsUniformInputs = 0; // Whether this operation requires that all - // of its inputs are uniform across the wave - // Group DXIL operation for stats - e.g., to accumulate the number of atomic/float/uint/int/... - // operations used in the program. - list<string> StatsGroup = []; -} - -class DXILOperation<string name, int opCode, DXILOpClass opClass, DXILOpCategory opCategory, string doc, - list<LLVMType> oloadTypes, EnumAttr attrs, list<DXILOpParameter> params, - list<string> statsGroup = []> : DXILOperationDesc { - let OpName = name; - let OpCode = opCode; - let Doc = doc; - let Params = params; - let OpClass = opClass; - let OpCategory = opCategory; - let OverloadTypes = oloadTypes; - let Attribute = attrs; - let StatsGroup = statsGroup; -} - -// LLVM intrinsic that DXIL operation maps to. -class LLVMIntrinsic<Intrinsic llvm_intrinsic_> { Intrinsic llvm_intrinsic = llvm_intrinsic_; } - -def Sin : DXILOperation<"Sin", 13, UnaryClass, UnaryFloatCategory, "returns sine(theta) for theta in radians.", - [llvm_half_ty, llvm_float_ty], ReadNone, - [ - DXILOpParameter<0, llvm_anyfloat_ty, "", "operation result">, - DXILOpParameter<1, llvm_i32_ty, "opcode", "DXIL opcode">, - DXILOpParameter<2, llvm_anyfloat_ty, "value", "input value"> - ], - ["floats"]>, - LLVMIntrinsic<int_sin>; - -def UMax : DXILOperation< "UMax", 39, BinaryClass, BinaryUintCategory, "unsigned integer maximum. UMax(a,b) = a > b ? a : b", - [llvm_i16_ty, llvm_i32_ty, llvm_i64_ty], ReadNone, - [ - DXILOpParameter<0, llvm_anyint_ty, "", "operation result">, - DXILOpParameter<1, llvm_i32_ty, "opcode", "DXIL opcode">, - DXILOpParameter<2, llvm_anyint_ty, "a", "input value">, - DXILOpParameter<3, llvm_anyint_ty, "b", "input value"> - ], - ["uints"]>, - LLVMIntrinsic<int_umax>; - -def ThreadId : DXILOperation< "ThreadId", 93, ThreadIdClass, ComputeIDCategory, "reads the thread ID", [llvm_i32_ty], ReadNone, - [ - DXILOpParameter<0, llvm_i32_ty, "", "thread ID component">, - DXILOpParameter<1, llvm_i32_ty, "opcode", "DXIL opcode">, - DXILOpParameter<2, llvm_i32_ty, "component", "component to read (x,y,z)"> - ]>, - LLVMIntrinsic<int_dx_thread_id>; - -def GroupId : DXILOperation< "GroupId", 94, GroupIdClass, ComputeIDCategory, "reads the group ID (SV_GroupID)", [llvm_i32_ty], ReadNone, - [ - DXILOpParameter<0, llvm_i32_ty, "", "group ID component">, - DXILOpParameter<1, llvm_i32_ty, "opcode", "DXIL opcode">, - DXILOpParameter<2, llvm_i32_ty, "component", "component to read"> - ]>, - LLVMIntrinsic<int_dx_group_id>; - -def ThreadIdInGroup : DXILOperation< "ThreadIdInGroup", 95, ThreadIdInGroupClass, ComputeIDCategory, - "reads the thread ID within the group (SV_GroupThreadID)", [llvm_i32_ty], ReadNone, - [ - DXILOpParameter<0, llvm_i32_ty, "", "thread ID in group component">, - DXILOpParameter<1, llvm_i32_ty, "opcode", "DXIL opcode">, - DXILOpParameter<2, llvm_i32_ty, "component", "component to read (x,y,z)"> - ]>, - LLVMIntrinsic<int_dx_thread_id_in_group>; - -def FlattenedThreadIdInGroup : DXILOperation< "FlattenedThreadIdInGroup", 96, FlattenedThreadIdInGroupClass, ComputeIDCategory, - "provides a flattened index for a given thread within a given group (SV_GroupIndex)", [llvm_i32_ty], ReadNone, - [ - DXILOpParameter<0, llvm_i32_ty, "", "result">, - DXILOpParameter<1, llvm_i32_ty, "opcode", "DXIL opcode"> - ]>, - LLVMIntrinsic<int_dx_flattened_thread_id_in_group>; +// Concrete definition of DXIL Operation mapping to corresponding LLVM intrinsic +def Sin : DXILOpMapping<13, unary, int_sin, + "Returns sine(theta) for theta in radians.">; +def UMax : DXILOpMapping<39, binary, int_umax, + "Unsigned integer maximum. UMax(a,b) = a > b ? a : b">; +def ThreadId : DXILOpMapping<93, threadId, int_dx_thread_id, + "Reads the thread ID">; +def GroupId : DXILOpMapping<94, groupId, int_dx_group_id, + "Reads the group ID (SV_GroupID)">; +def ThreadIdInGroup : DXILOpMapping<95, threadIdInGroup, + int_dx_thread_id_in_group, + "Reads the thread ID within the group " + "(SV_GroupThreadID)">; +def FlattenedThreadIdInGroup : DXILOpMapping<96, flattenedThreadIdInGroup, + int_dx_flattened_thread_id_in_group, + "Provides a flattened index for a " + "given thread within a given " + "group (SV_GroupIndex)">; diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp index 42180a8..21a20d4 100644 --- a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp +++ b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp @@ -221,12 +221,26 @@ static Type *getTypeFromParameterKind(ParameterKind Kind, Type *OverloadTy) { return nullptr; } +/// Construct DXIL function type. This is the type of a function with +/// the following prototype +/// OverloadType dx.op.<opclass>.<return-type>(int opcode, <param types>) +/// <param-types> are constructed from types in Prop. +/// \param Prop Structure containing DXIL Operation properties based on +/// its specification in DXIL.td. +/// \param OverloadTy Return type to be used to construct DXIL function type. static FunctionType *getDXILOpFunctionType(const OpCodeProperty *Prop, Type *OverloadTy) { SmallVector<Type *> ArgTys; auto ParamKinds = getOpCodeParameterKind(*Prop); + // Add OverloadTy as return type of the function + ArgTys.emplace_back(OverloadTy); + + // Add DXIL Opcode value type viz., Int32 as first argument + ArgTys.emplace_back(Type::getInt32Ty(OverloadTy->getContext())); + + // Add DXIL Operation parameter types as specified in DXIL properties for (unsigned I = 0; I < Prop->NumOfParameters; ++I) { ParameterKind Kind = ParamKinds[I]; ArgTys.emplace_back(getTypeFromParameterKind(Kind, OverloadTy)); @@ -267,13 +281,13 @@ CallInst *DXILOpBuilder::createDXILOpCall(dxil::OpCode OpCode, Type *OverloadTy, return B.CreateCall(Fn, FullArgs); } -Type *DXILOpBuilder::getOverloadTy(dxil::OpCode OpCode, FunctionType *FT, - bool NoOpCodeParam) { +Type *DXILOpBuilder::getOverloadTy(dxil::OpCode OpCode, FunctionType *FT) { const OpCodeProperty *Prop = getOpCodeProperty(OpCode); + // If DXIL Op has no overload parameter, just return the + // precise return type specified. if (Prop->OverloadParamIndex < 0) { auto &Ctx = FT->getContext(); - // When only has 1 overload type, just return it. switch (Prop->OverloadTys) { case OverloadKind::VOID: return Type::getVoidTy(Ctx); @@ -302,9 +316,8 @@ Type *DXILOpBuilder::getOverloadTy(dxil::OpCode OpCode, FunctionType *FT, // Prop->OverloadParamIndex is 0, overload type is FT->getReturnType(). Type *OverloadType = FT->getReturnType(); if (Prop->OverloadParamIndex != 0) { - // Skip Return Type and Type for DXIL opcode. - const unsigned SkipedParam = NoOpCodeParam ? 2 : 1; - OverloadType = FT->getParamType(Prop->OverloadParamIndex - SkipedParam); + // Skip Return Type. + OverloadType = FT->getParamType(Prop->OverloadParamIndex - 1); } auto ParamKinds = getOpCodeParameterKind(*Prop); diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.h b/llvm/lib/Target/DirectX/DXILOpBuilder.h index 940ed53..1c15f10 100644 --- a/llvm/lib/Target/DirectX/DXILOpBuilder.h +++ b/llvm/lib/Target/DirectX/DXILOpBuilder.h @@ -31,8 +31,7 @@ public: DXILOpBuilder(Module &M, IRBuilderBase &B) : M(M), B(B) {} CallInst *createDXILOpCall(dxil::OpCode OpCode, Type *OverloadTy, llvm::iterator_range<Use *> Args); - Type *getOverloadTy(dxil::OpCode OpCode, FunctionType *FT, - bool NoOpCodeParam); + Type *getOverloadTy(dxil::OpCode OpCode, FunctionType *FT); static const char *getOpCodeName(dxil::OpCode DXILOp); private: diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp index f6e2297..6b649b7 100644 --- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp +++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp @@ -33,8 +33,7 @@ static void lowerIntrinsic(dxil::OpCode DXILOp, Function &F, Module &M) { IRBuilder<> B(M.getContext()); Value *DXILOpArg = B.getInt32(static_cast<unsigned>(DXILOp)); DXILOpBuilder DXILB(M, B); - Type *OverloadTy = - DXILB.getOverloadTy(DXILOp, F.getFunctionType(), /*NoOpCodeParam*/ true); + Type *OverloadTy = DXILB.getOverloadTy(DXILOp, F.getFunctionType()); for (User *U : make_early_inc_range(F.users())) { CallInst *CI = dyn_cast<CallInst>(U); if (!CI) diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index bcaf447..9773b29 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -185,7 +185,7 @@ def HasStdExtZabha : Predicate<"Subtarget->hasStdExtZabha()">, "'Zabha' (Byte and Halfword Atomic Memory Operations)">; def FeatureStdExtZacas - : SubtargetFeature<"experimental-zacas", "HasStdExtZacas", "true", + : SubtargetFeature<"zacas", "HasStdExtZacas", "true", "'Zacas' (Atomic Compare-And-Swap Instructions)">; def HasStdExtZacas : Predicate<"Subtarget->hasStdExtZacas()">, AssemblerPredicate<(all_of FeatureStdExtZacas), diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index dde1882..e647f56 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -1999,6 +1999,10 @@ bool RISCVTargetLowering::canSplatOperand(Instruction *I, int Operand) const { case Intrinsic::vp_sdiv: case Intrinsic::vp_urem: case Intrinsic::vp_srem: + case Intrinsic::ssub_sat: + case Intrinsic::vp_ssub_sat: + case Intrinsic::usub_sat: + case Intrinsic::vp_usub_sat: return Operand == 1; // These intrinsics are commutative. case Intrinsic::vp_add: @@ -2010,6 +2014,18 @@ bool RISCVTargetLowering::canSplatOperand(Instruction *I, int Operand) const { case Intrinsic::vp_fmul: case Intrinsic::vp_icmp: case Intrinsic::vp_fcmp: + case Intrinsic::smin: + case Intrinsic::vp_smin: + case Intrinsic::umin: + case Intrinsic::vp_umin: + case Intrinsic::smax: + case Intrinsic::vp_smax: + case Intrinsic::umax: + case Intrinsic::vp_umax: + case Intrinsic::sadd_sat: + case Intrinsic::vp_sadd_sat: + case Intrinsic::uadd_sat: + case Intrinsic::vp_uadd_sat: // These intrinsics have 'vr' versions. case Intrinsic::vp_sub: case Intrinsic::vp_fsub: diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp index 9d1f01d..a68674b 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp @@ -30,6 +30,8 @@ using namespace llvm; +static cl::opt<bool> DisableCostPerUse("riscv-disable-cost-per-use", + cl::init(false), cl::Hidden); static cl::opt<bool> DisableRegAllocHints("riscv-disable-regalloc-hints", cl::Hidden, cl::init(false), @@ -712,7 +714,10 @@ void RISCVRegisterInfo::getOffsetOpcodes(const StackOffset &Offset, unsigned RISCVRegisterInfo::getRegisterCostTableIndex(const MachineFunction &MF) const { - return MF.getSubtarget<RISCVSubtarget>().hasStdExtCOrZca() ? 1 : 0; + return MF.getSubtarget<RISCVSubtarget>().hasStdExtCOrZca() && + !DisableCostPerUse + ? 1 + : 0; } // Add two address hints to improve chances of being able to use a compressed diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td index 040cec4..0430d60 100644 --- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td +++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td @@ -198,6 +198,7 @@ def SiFive7Model : SchedMachineModel { let LoadLatency = 3; let MispredictPenalty = 3; let CompleteModel = 0; + let PostRAScheduler = true; let EnableIntervals = true; let UnsupportedFeatures = [HasStdExtZbkb, HasStdExtZbkc, HasStdExtZbkx, HasStdExtZcmt, HasStdExtZknd, HasStdExtZkne, diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index d23921f..a0b55b2 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -326,18 +326,6 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, switch (Kind) { default: break; - case TTI::SK_InsertSubvector: { - auto *FSubTy = cast<FixedVectorType>(SubTp); - unsigned TpRegs = getRegUsageForType(Tp); - unsigned SubTpRegs = getRegUsageForType(SubTp); - unsigned NextSubTpRegs = getRegUsageForType(FixedVectorType::get( - Tp->getElementType(), FSubTy->getNumElements() + 1)); - // Whole vector insert - just the vector itself. - if (Index == 0 && SubTpRegs != 0 && SubTpRegs != NextSubTpRegs && - TpRegs >= SubTpRegs) - return TTI::TCC_Free; - break; - } case TTI::SK_PermuteSingleSrc: { if (Mask.size() >= 2 && LT.second.isFixedLengthVector()) { MVT EltTp = LT.second.getVectorElementType(); @@ -469,15 +457,28 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, // vslidedown.vi v8, v9, 2 return LT.first * getRISCVInstructionCost(RISCV::VSLIDEDOWN_VI, LT.second, CostKind); - case TTI::SK_InsertSubvector: + case TTI::SK_InsertSubvector: { if (Index == 0 && !Args.empty() && any_of(Args, UndefValue::classof)) return TTI::TCC_Free; + const unsigned MinVLen = ST->getRealMinVLen(); + const unsigned MaxVLen = ST->getRealMaxVLen(); + // Whole vector insert - just the vector itself. + if (auto *FSubTy = dyn_cast<FixedVectorType>(SubTp); + FSubTy && Index == 0 && MinVLen == MaxVLen) { + unsigned TpRegs = getRegUsageForType(Tp); + unsigned SubTpRegs = getRegUsageForType(SubTp); + unsigned NextSubTpRegs = getRegUsageForType(FixedVectorType::get( + Tp->getElementType(), FSubTy->getNumElements() + 1)); + if (SubTpRegs != 0 && SubTpRegs != NextSubTpRegs && TpRegs >= SubTpRegs) + return TTI::TCC_Free; + } // Example sequence: // vsetivli zero, 4, e8, mf2, tu, ma (ignored) // vslideup.vi v8, v9, 2 return LT.first * getRISCVInstructionCost(RISCV::VSLIDEUP_VI, LT.second, CostKind); + } case TTI::SK_Select: { // Example sequence: // li a0, 90 @@ -503,9 +504,8 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, // vmv.v.x v8, a0 // vmsne.vi v0, v8, 0 return LT.first * - (TLI->getLMULCost(LT.second) + // FIXME: should be 1 for andi - getRISCVInstructionCost({RISCV::VMV_V_X, RISCV::VMSNE_VI}, - LT.second, CostKind)); + (1 + getRISCVInstructionCost({RISCV::VMV_V_X, RISCV::VMSNE_VI}, + LT.second, CostKind)); } // Example sequence: // vsetivli zero, 2, e8, mf8, ta, mu (ignored) @@ -517,11 +517,10 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, // vmsne.vi v0, v8, 0 return LT.first * - (TLI->getLMULCost(LT.second) + // FIXME: this should be 1 for andi - getRISCVInstructionCost({RISCV::VMV_V_I, RISCV::VMERGE_VIM, - RISCV::VMV_X_S, RISCV::VMV_V_X, - RISCV::VMSNE_VI}, - LT.second, CostKind)); + (1 + getRISCVInstructionCost({RISCV::VMV_V_I, RISCV::VMERGE_VIM, + RISCV::VMV_X_S, RISCV::VMV_V_X, + RISCV::VMSNE_VI}, + LT.second, CostKind)); } if (HasScalar) { @@ -566,9 +565,12 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, if (LT.second.isFixedLengthVector()) // vrsub.vi has a 5 bit immediate field, otherwise an li suffices LenCost = isInt<5>(LT.second.getVectorNumElements() - 1) ? 0 : 1; - // FIXME: replace the constant `2` below with cost of {VID_V,VRSUB_VX} + unsigned Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX, RISCV::VRGATHER_VV}; + if (LT.second.isFixedLengthVector() && + isInt<5>(LT.second.getVectorNumElements() - 1)) + Opcodes[1] = RISCV::VRSUB_VI; InstructionCost GatherCost = - 2 + getRISCVInstructionCost(RISCV::VRGATHER_VV, LT.second, CostKind); + getRISCVInstructionCost(Opcodes, LT.second, CostKind); // Mask operation additionally required extend and truncate InstructionCost ExtendCost = Tp->getElementType()->isIntegerTy(1) ? 3 : 0; return LT.first * (LenCost + GatherCost + ExtendCost); diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp index 95c4b02..b9fb3fd 100644 --- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp +++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp @@ -61,22 +61,22 @@ public: } private: - void ExpandICallBranchFunnel(MachineBasicBlock *MBB, + void expandICallBranchFunnel(MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI); void expandCALL_RVMARKER(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI); - bool ExpandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI); - bool ExpandMBB(MachineBasicBlock &MBB); + bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI); + bool expandMBB(MachineBasicBlock &MBB); /// This function expands pseudos which affects control flow. /// It is done in separate pass to simplify blocks navigation in main - /// pass(calling ExpandMBB). - bool ExpandPseudosWhichAffectControlFlow(MachineFunction &MF); + /// pass(calling expandMBB). + bool expandPseudosWhichAffectControlFlow(MachineFunction &MF); /// Expand X86::VASTART_SAVE_XMM_REGS into set of xmm copying instructions, /// placed into separate block guarded by check for al register(for SystemV /// abi). - void ExpandVastartSaveXmmRegs( + void expandVastartSaveXmmRegs( MachineBasicBlock *EntryBlk, MachineBasicBlock::iterator VAStartPseudoInstr) const; }; @@ -87,7 +87,7 @@ char X86ExpandPseudo::ID = 0; INITIALIZE_PASS(X86ExpandPseudo, DEBUG_TYPE, X86_EXPAND_PSEUDO_NAME, false, false) -void X86ExpandPseudo::ExpandICallBranchFunnel( +void X86ExpandPseudo::expandICallBranchFunnel( MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI) { MachineBasicBlock *JTMBB = MBB; MachineInstr *JTInst = &*MBBI; @@ -259,12 +259,12 @@ void X86ExpandPseudo::expandCALL_RVMARKER(MachineBasicBlock &MBB, /// If \p MBBI is a pseudo instruction, this method expands /// it to the corresponding (sequence of) actual instruction(s). /// \returns true if \p MBBI has been expanded. -bool X86ExpandPseudo::ExpandMI(MachineBasicBlock &MBB, +bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) { MachineInstr &MI = *MBBI; unsigned Opcode = MI.getOpcode(); const DebugLoc &DL = MBBI->getDebugLoc(); - bool HasEGPR = STI->hasEGPR(); +#define GET_EGPR_IF_ENABLED(OPC) (STI->hasEGPR() ? OPC##_EVEX : OPC) switch (Opcode) { default: return false; @@ -468,12 +468,10 @@ bool X86ExpandPseudo::ExpandMI(MachineBasicBlock &MBB, Register Reg1 = TRI->getSubReg(Reg, X86::sub_mask_1); auto MIBLo = - BuildMI(MBB, MBBI, DL, - TII->get(HasEGPR ? X86::KMOVWkm_EVEX : X86::KMOVWkm)) + BuildMI(MBB, MBBI, DL, TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWkm))) .addReg(Reg0, RegState::Define | getDeadRegState(DstIsDead)); auto MIBHi = - BuildMI(MBB, MBBI, DL, - TII->get(HasEGPR ? X86::KMOVWkm_EVEX : X86::KMOVWkm)) + BuildMI(MBB, MBBI, DL, TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWkm))) .addReg(Reg1, RegState::Define | getDeadRegState(DstIsDead)); for (int i = 0; i < X86::AddrNumOperands; ++i) { @@ -505,10 +503,10 @@ bool X86ExpandPseudo::ExpandMI(MachineBasicBlock &MBB, Register Reg0 = TRI->getSubReg(Reg, X86::sub_mask_0); Register Reg1 = TRI->getSubReg(Reg, X86::sub_mask_1); - auto MIBLo = BuildMI(MBB, MBBI, DL, - TII->get(HasEGPR ? X86::KMOVWmk_EVEX : X86::KMOVWmk)); - auto MIBHi = BuildMI(MBB, MBBI, DL, - TII->get(HasEGPR ? X86::KMOVWmk_EVEX : X86::KMOVWmk)); + auto MIBLo = + BuildMI(MBB, MBBI, DL, TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWmk))); + auto MIBHi = + BuildMI(MBB, MBBI, DL, TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWmk))); for (int i = 0; i < X86::AddrNumOperands; ++i) { MIBLo.add(MBBI->getOperand(i)); @@ -554,9 +552,8 @@ bool X86ExpandPseudo::ExpandMI(MachineBasicBlock &MBB, return true; } case TargetOpcode::ICALL_BRANCH_FUNNEL: - ExpandICallBranchFunnel(&MBB, MBBI); + expandICallBranchFunnel(&MBB, MBBI); return true; -#define GET_EGPR_IF_ENABLED(OPC) (STI->hasEGPR() ? OPC##_EVEX : OPC) case X86::PLDTILECFGV: { MI.setDesc(TII->get(GET_EGPR_IF_ENABLED(X86::LDTILECFG))); return true; @@ -634,7 +631,7 @@ bool X86ExpandPseudo::ExpandMI(MachineBasicBlock &MBB, // | | // | | // -void X86ExpandPseudo::ExpandVastartSaveXmmRegs( +void X86ExpandPseudo::expandVastartSaveXmmRegs( MachineBasicBlock *EntryBlk, MachineBasicBlock::iterator VAStartPseudoInstr) const { assert(VAStartPseudoInstr->getOpcode() == X86::VASTART_SAVE_XMM_REGS); @@ -719,27 +716,27 @@ void X86ExpandPseudo::ExpandVastartSaveXmmRegs( /// Expand all pseudo instructions contained in \p MBB. /// \returns true if any expansion occurred for \p MBB. -bool X86ExpandPseudo::ExpandMBB(MachineBasicBlock &MBB) { +bool X86ExpandPseudo::expandMBB(MachineBasicBlock &MBB) { bool Modified = false; // MBBI may be invalidated by the expansion. MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); while (MBBI != E) { MachineBasicBlock::iterator NMBBI = std::next(MBBI); - Modified |= ExpandMI(MBB, MBBI); + Modified |= expandMI(MBB, MBBI); MBBI = NMBBI; } return Modified; } -bool X86ExpandPseudo::ExpandPseudosWhichAffectControlFlow(MachineFunction &MF) { +bool X86ExpandPseudo::expandPseudosWhichAffectControlFlow(MachineFunction &MF) { // Currently pseudo which affects control flow is only // X86::VASTART_SAVE_XMM_REGS which is located in Entry block. // So we do not need to evaluate other blocks. for (MachineInstr &Instr : MF.front().instrs()) { if (Instr.getOpcode() == X86::VASTART_SAVE_XMM_REGS) { - ExpandVastartSaveXmmRegs(&(MF.front()), Instr); + expandVastartSaveXmmRegs(&(MF.front()), Instr); return true; } } @@ -754,10 +751,10 @@ bool X86ExpandPseudo::runOnMachineFunction(MachineFunction &MF) { X86FI = MF.getInfo<X86MachineFunctionInfo>(); X86FL = STI->getFrameLowering(); - bool Modified = ExpandPseudosWhichAffectControlFlow(MF); + bool Modified = expandPseudosWhichAffectControlFlow(MF); for (MachineBasicBlock &MBB : MF) - Modified |= ExpandMBB(MBB); + Modified |= expandMBB(MBB); return Modified; } diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp index c8f80ce..5cbd9ab 100644 --- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp +++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp @@ -2732,13 +2732,15 @@ bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM, insertDAGNode(*CurDAG, N, Zext); SDValue NewShl = CurDAG->getNode(ISD::SHL, DL, VT, Zext, ShlAmt); insertDAGNode(*CurDAG, N, NewShl); + CurDAG->ReplaceAllUsesWith(N, NewShl); + CurDAG->RemoveDeadNode(N.getNode()); // Convert the shift to scale factor. AM.Scale = 1 << ShAmtV; - AM.IndexReg = Zext; - - CurDAG->ReplaceAllUsesWith(N, NewShl); - CurDAG->RemoveDeadNode(N.getNode()); + // If matchIndexRecursively is not called here, + // Zext may be replaced by other nodes but later used to call a builder + // method + AM.IndexReg = matchIndexRecursively(Zext, AM, Depth + 1); return false; } diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index bec13d1..d98d914 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -5878,13 +5878,16 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts, } } - // Peek through trunc/aext/zext. + // Peek through trunc/aext/zext/bitcast. // TODO: aext shouldn't require SM_SentinelZero padding. // TODO: handle shift of scalars. unsigned MinBitsPerElt = Scl.getScalarValueSizeInBits(); while (Scl.getOpcode() == ISD::TRUNCATE || Scl.getOpcode() == ISD::ANY_EXTEND || - Scl.getOpcode() == ISD::ZERO_EXTEND) { + Scl.getOpcode() == ISD::ZERO_EXTEND || + (Scl.getOpcode() == ISD::BITCAST && + Scl.getScalarValueSizeInBits() == + Scl.getOperand(0).getScalarValueSizeInBits())) { Scl = Scl.getOperand(0); MinBitsPerElt = std::min<unsigned>(MinBitsPerElt, Scl.getScalarValueSizeInBits()); @@ -41399,7 +41402,9 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( break; } case X86ISD::CVTSI2P: - case X86ISD::CVTUI2P: { + case X86ISD::CVTUI2P: + case X86ISD::CVTPH2PS: + case X86ISD::CVTPS2PH: { SDValue Src = Op.getOperand(0); MVT SrcVT = Src.getSimpleValueType(); APInt SrcUndef, SrcZero; diff --git a/llvm/lib/Target/X86/X86InstrSystem.td b/llvm/lib/Target/X86/X86InstrSystem.td index d051047..56293e2 100644 --- a/llvm/lib/Target/X86/X86InstrSystem.td +++ b/llvm/lib/Target/X86/X86InstrSystem.td @@ -716,7 +716,7 @@ def INVPCID64 : I<0x82, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), def INVPCID64_EVEX : I<0xF2, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invpcid\t{$src2, $src1|$src1, $src2}", []>, - EVEX, NoCD8, T_MAP4, XS, Requires<[In64BitMode]>; + EVEX, NoCD8, T_MAP4, XS, WIG, Requires<[In64BitMode]>; } // SchedRW let Predicates = [HasINVPCID, NoEGPR] in { diff --git a/llvm/lib/Target/X86/X86InstrUtils.td b/llvm/lib/Target/X86/X86InstrUtils.td index 05ddcfb..04d9d10 100644 --- a/llvm/lib/Target/X86/X86InstrUtils.td +++ b/llvm/lib/Target/X86/X86InstrUtils.td @@ -967,6 +967,7 @@ class ITy<bits<8> o, Format f, X86TypeInfo t, dag outs, dag ins, string m, !strconcat(m, "{", t.InstrSuffix, "}\t", args), p>, NoCD8 { let hasSideEffects = 0; let hasREX_W = t.HasREX_W; + let IgnoresW = !if(!eq(t.VT, i8), 1, 0); } // BinOpRR - Instructions that read "reg, reg". diff --git a/llvm/lib/Target/X86/X86InstrVMX.td b/llvm/lib/Target/X86/X86InstrVMX.td index 7cc468f..da2b3d7 100644 --- a/llvm/lib/Target/X86/X86InstrVMX.td +++ b/llvm/lib/Target/X86/X86InstrVMX.td @@ -24,7 +24,7 @@ def INVEPT64 : I<0x80, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), Requires<[In64BitMode]>; def INVEPT64_EVEX : I<0xF0, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invept\t{$src2, $src1|$src1, $src2}", []>, - EVEX, NoCD8, T_MAP4, XS, Requires<[In64BitMode]>; + EVEX, NoCD8, T_MAP4, XS, WIG, Requires<[In64BitMode]>; // 66 0F 38 81 def INVVPID32 : I<0x81, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2), @@ -35,7 +35,7 @@ def INVVPID64 : I<0x81, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), Requires<[In64BitMode]>; def INVVPID64_EVEX : I<0xF1, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invvpid\t{$src2, $src1|$src1, $src2}", []>, - EVEX, NoCD8, T_MAP4, XS, Requires<[In64BitMode]>; + EVEX, NoCD8, T_MAP4, XS, WIG, Requires<[In64BitMode]>; // 0F 01 C1 def VMCALL : I<0x01, MRM_C1, (outs), (ins), "vmcall", []>, TB; diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp index 49b3f2b..5c7a74d 100644 --- a/llvm/lib/Transforms/IPO/FunctionImport.cpp +++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp @@ -125,7 +125,8 @@ static cl::opt<bool> ComputeDead("compute-dead", cl::init(true), cl::Hidden, static cl::opt<bool> EnableImportMetadata( "enable-import-metadata", cl::init(false), cl::Hidden, - cl::desc("Enable import metadata like 'thinlto_src_module'")); + cl::desc("Enable import metadata like 'thinlto_src_module' and " + "'thinlto_src_file'")); /// Summary file to use for function importing when using -function-import from /// the command line. @@ -1643,11 +1644,17 @@ Expected<bool> FunctionImporter::importFunctions( if (Error Err = F.materialize()) return std::move(Err); if (EnableImportMetadata) { - // Add 'thinlto_src_module' metadata for statistics and debugging. + // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for + // statistics and debugging. F.setMetadata( "thinlto_src_module", MDNode::get(DestModule.getContext(), {MDString::get(DestModule.getContext(), + SrcModule->getModuleIdentifier())})); + F.setMetadata( + "thinlto_src_file", + MDNode::get(DestModule.getContext(), + {MDString::get(DestModule.getContext(), SrcModule->getSourceFileName())})); } GlobalsToImport.insert(&F); @@ -1687,11 +1694,17 @@ Expected<bool> FunctionImporter::importFunctions( << GO->getName() << " from " << SrcModule->getSourceFileName() << "\n"); if (EnableImportMetadata) { - // Add 'thinlto_src_module' metadata for statistics and debugging. + // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for + // statistics and debugging. Fn->setMetadata( "thinlto_src_module", MDNode::get(DestModule.getContext(), {MDString::get(DestModule.getContext(), + SrcModule->getModuleIdentifier())})); + Fn->setMetadata( + "thinlto_src_file", + MDNode::get(DestModule.getContext(), + {MDString::get(DestModule.getContext(), SrcModule->getSourceFileName())})); } GlobalsToImport.insert(Fn); diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp index 6ce9eb3..490cb7e 100644 --- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp +++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp @@ -905,8 +905,8 @@ static bool processSRem(BinaryOperator *SDI, const ConstantRange &LCR, for (Operand &Op : Ops) { if (Op.D == Domain::NonNegative) continue; - auto *BO = - BinaryOperator::CreateNeg(Op.V, Op.V->getName() + ".nonneg", SDI); + auto *BO = BinaryOperator::CreateNeg(Op.V, Op.V->getName() + ".nonneg", + SDI->getIterator()); BO->setDebugLoc(SDI->getDebugLoc()); Op.V = BO; } @@ -919,7 +919,8 @@ static bool processSRem(BinaryOperator *SDI, const ConstantRange &LCR, // If the divident was non-positive, we need to negate the result. if (Ops[0].D == Domain::NonPositive) { - Res = BinaryOperator::CreateNeg(Res, Res->getName() + ".neg", SDI); + Res = BinaryOperator::CreateNeg(Res, Res->getName() + ".neg", + SDI->getIterator()); Res->setDebugLoc(SDI->getDebugLoc()); } @@ -966,8 +967,8 @@ static bool processSDiv(BinaryOperator *SDI, const ConstantRange &LCR, for (Operand &Op : Ops) { if (Op.D == Domain::NonNegative) continue; - auto *BO = - BinaryOperator::CreateNeg(Op.V, Op.V->getName() + ".nonneg", SDI); + auto *BO = BinaryOperator::CreateNeg(Op.V, Op.V->getName() + ".nonneg", + SDI->getIterator()); BO->setDebugLoc(SDI->getDebugLoc()); Op.V = BO; } @@ -981,7 +982,8 @@ static bool processSDiv(BinaryOperator *SDI, const ConstantRange &LCR, // If the operands had two different domains, we need to negate the result. if (Ops[0].D != Ops[1].D) { - Res = BinaryOperator::CreateNeg(Res, Res->getName() + ".neg", SDI); + Res = BinaryOperator::CreateNeg(Res, Res->getName() + ".neg", + SDI->getIterator()); Res->setDebugLoc(SDI->getDebugLoc()); } diff --git a/llvm/lib/Transforms/Scalar/Reassociate.cpp b/llvm/lib/Transforms/Scalar/Reassociate.cpp index 818c7b4..61109ed 100644 --- a/llvm/lib/Transforms/Scalar/Reassociate.cpp +++ b/llvm/lib/Transforms/Scalar/Reassociate.cpp @@ -270,7 +270,8 @@ static BinaryOperator *CreateMul(Value *S1, Value *S2, const Twine &Name, } static Instruction *CreateNeg(Value *S1, const Twine &Name, - Instruction *InsertBefore, Value *FlagsOp) { + BasicBlock::iterator InsertBefore, + Value *FlagsOp) { if (S1->getType()->isIntOrIntVectorTy()) return BinaryOperator::CreateNeg(S1, Name, InsertBefore); @@ -958,7 +959,8 @@ static Value *NegateValue(Value *V, Instruction *BI, // Insert a 'neg' instruction that subtracts the value from zero to get the // negation. - Instruction *NewNeg = CreateNeg(V, V->getName() + ".neg", BI, BI); + Instruction *NewNeg = + CreateNeg(V, V->getName() + ".neg", BI->getIterator(), BI); ToRedo.insert(NewNeg); return NewNeg; } @@ -1246,7 +1248,7 @@ Value *ReassociatePass::RemoveFactorFromExpression(Value *V, Value *Factor) { } if (NeedsNegate) - V = CreateNeg(V, "neg", &*InsertPt, BO); + V = CreateNeg(V, "neg", InsertPt, BO); return V; } diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index ea77b60..50a073e 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -315,12 +315,6 @@ static cl::opt<bool> EnableLoadStoreRuntimeInterleave( cl::desc( "Enable runtime interleaving until load/store ports are saturated")); -/// Interleave small loops with scalar reductions. -static cl::opt<bool> InterleaveSmallLoopScalarReduction( - "interleave-small-loop-scalar-reduction", cl::init(false), cl::Hidden, - cl::desc("Enable interleaving for loops with small iteration counts that " - "contain scalar reductions to expose ILP.")); - /// The number of stores in a loop that are allowed to need predication. static cl::opt<unsigned> NumberOfStoresToPredicate( "vectorize-num-stores-pred", cl::init(1), cl::Hidden, @@ -5495,8 +5489,7 @@ LoopVectorizationCostModel::selectInterleaveCount(ElementCount VF, // If there are scalar reductions and TTI has enabled aggressive // interleaving for reductions, we will interleave to expose ILP. - if (InterleaveSmallLoopScalarReduction && VF.isScalar() && - AggressivelyInterleaveReductions) { + if (VF.isScalar() && AggressivelyInterleaveReductions) { LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n"); // Interleave no less than SmallIC but not as aggressive as the normal IC // to satisfy the rare situation when resources are too limited. @@ -9010,11 +9003,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( BasicBlock *BB = CurrentLinkI->getParent(); VPValue *CondOp = nullptr; - if (CM.blockNeedsPredicationForAnyReason(BB)) { - VPBuilder::InsertPointGuard Guard(Builder); - Builder.setInsertPoint(CurrentLink); + if (CM.blockNeedsPredicationForAnyReason(BB)) CondOp = RecipeBuilder.getBlockInMask(BB); - } VPReductionRecipe *RedRecipe = new VPReductionRecipe( RdxDesc, CurrentLinkI, PreviousLink, VecOp, CondOp); diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-shuffle.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-shuffle.ll index 30da63b..7cc7cff 100644 --- a/llvm/test/Analysis/CostModel/RISCV/rvv-shuffle.ll +++ b/llvm/test/Analysis/CostModel/RISCV/rvv-shuffle.ll @@ -14,7 +14,7 @@ define void @vector_broadcast() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = shufflevector <vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = shufflevector <vscale x 1 x i64> undef, <vscale x 1 x i64> undef, <vscale x 1 x i32> zeroinitializer ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = shufflevector <vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer -; CHECK-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %8 = shufflevector <vscale x 16 x i1> undef, <vscale x 16 x i1> undef, <vscale x 16 x i32> zeroinitializer +; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %8 = shufflevector <vscale x 16 x i1> undef, <vscale x 16 x i1> undef, <vscale x 16 x i32> zeroinitializer ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %9 = shufflevector <vscale x 8 x i1> undef, <vscale x 8 x i1> undef, <vscale x 8 x i32> zeroinitializer ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %10 = shufflevector <vscale x 4 x i1> undef, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %11 = shufflevector <vscale x 2 x i1> undef, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer @@ -29,7 +29,7 @@ define void @vector_broadcast() { ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = shufflevector <vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = shufflevector <vscale x 1 x i64> undef, <vscale x 1 x i64> undef, <vscale x 1 x i32> zeroinitializer ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = shufflevector <vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer -; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %8 = shufflevector <vscale x 16 x i1> undef, <vscale x 16 x i1> undef, <vscale x 16 x i32> zeroinitializer +; SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %8 = shufflevector <vscale x 16 x i1> undef, <vscale x 16 x i1> undef, <vscale x 16 x i32> zeroinitializer ; SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %9 = shufflevector <vscale x 8 x i1> undef, <vscale x 8 x i1> undef, <vscale x 8 x i32> zeroinitializer ; SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %10 = shufflevector <vscale x 4 x i1> undef, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer ; SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %11 = shufflevector <vscale x 2 x i1> undef, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer @@ -78,20 +78,20 @@ declare <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x define void @vector_reverse() { ; CHECK-LABEL: 'vector_reverse' -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %reverse_nxv16i8 = call <vscale x 16 x i8> @llvm.experimental.vector.reverse.nxv16i8(<vscale x 16 x i8> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %reverse_nxv32i8 = call <vscale x 32 x i8> @llvm.experimental.vector.reverse.nxv32i8(<vscale x 32 x i8> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %reverse_nxv16i8 = call <vscale x 16 x i8> @llvm.experimental.vector.reverse.nxv16i8(<vscale x 16 x i8> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %reverse_nxv32i8 = call <vscale x 32 x i8> @llvm.experimental.vector.reverse.nxv32i8(<vscale x 32 x i8> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %reverse_nxv2i16 = call <vscale x 2 x i16> @llvm.experimental.vector.reverse.nxv2i16(<vscale x 2 x i16> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %reverse_nxv4i16 = call <vscale x 4 x i16> @llvm.experimental.vector.reverse.nxv4i16(<vscale x 4 x i16> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %reverse_nxv8i16 = call <vscale x 8 x i16> @llvm.experimental.vector.reverse.nxv8i16(<vscale x 8 x i16> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %reverse_nxv16i16 = call <vscale x 16 x i16> @llvm.experimental.vector.reverse.nxv16i16(<vscale x 16 x i16> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %reverse_nxv4i32 = call <vscale x 4 x i32> @llvm.experimental.vector.reverse.nxv4i32(<vscale x 4 x i32> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %reverse_nxv8i32 = call <vscale x 8 x i32> @llvm.experimental.vector.reverse.nxv8i32(<vscale x 8 x i32> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %reverse_nxv2i64 = call <vscale x 2 x i64> @llvm.experimental.vector.reverse.nxv2i64(<vscale x 2 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %reverse_nxv4i64 = call <vscale x 4 x i64> @llvm.experimental.vector.reverse.nxv4i64(<vscale x 4 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 69 for instruction: %reverse_nxv8i64 = call <vscale x 8 x i64> @llvm.experimental.vector.reverse.nxv8i64(<vscale x 8 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 138 for instruction: %reverse_nxv16i64 = call <vscale x 16 x i64> @llvm.experimental.vector.reverse.nxv16i64(<vscale x 16 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 276 for instruction: %reverse_nxv32i64 = call <vscale x 32 x i64> @llvm.experimental.vector.reverse.nxv32i64(<vscale x 32 x i64> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %reverse_nxv16i1 = call <vscale x 16 x i1> @llvm.experimental.vector.reverse.nxv16i1(<vscale x 16 x i1> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %reverse_nxv8i16 = call <vscale x 8 x i16> @llvm.experimental.vector.reverse.nxv8i16(<vscale x 8 x i16> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %reverse_nxv16i16 = call <vscale x 16 x i16> @llvm.experimental.vector.reverse.nxv16i16(<vscale x 16 x i16> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %reverse_nxv4i32 = call <vscale x 4 x i32> @llvm.experimental.vector.reverse.nxv4i32(<vscale x 4 x i32> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %reverse_nxv8i32 = call <vscale x 8 x i32> @llvm.experimental.vector.reverse.nxv8i32(<vscale x 8 x i32> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %reverse_nxv2i64 = call <vscale x 2 x i64> @llvm.experimental.vector.reverse.nxv2i64(<vscale x 2 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %reverse_nxv4i64 = call <vscale x 4 x i64> @llvm.experimental.vector.reverse.nxv4i64(<vscale x 4 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 83 for instruction: %reverse_nxv8i64 = call <vscale x 8 x i64> @llvm.experimental.vector.reverse.nxv8i64(<vscale x 8 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 166 for instruction: %reverse_nxv16i64 = call <vscale x 16 x i64> @llvm.experimental.vector.reverse.nxv16i64(<vscale x 16 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 332 for instruction: %reverse_nxv32i64 = call <vscale x 32 x i64> @llvm.experimental.vector.reverse.nxv32i64(<vscale x 32 x i64> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %reverse_nxv16i1 = call <vscale x 16 x i1> @llvm.experimental.vector.reverse.nxv16i1(<vscale x 16 x i1> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %reverse_nxv8i1 = call <vscale x 8 x i1> @llvm.experimental.vector.reverse.nxv8i1(<vscale x 8 x i1> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %reverse_nxv4i1 = call <vscale x 4 x i1> @llvm.experimental.vector.reverse.nxv4i1(<vscale x 4 x i1> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %reverse_nxv2i1 = call <vscale x 2 x i1> @llvm.experimental.vector.reverse.nxv2i1(<vscale x 2 x i1> undef) diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-vectorinsert.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-vectorinsert.ll index 5b16b87..607abee 100644 --- a/llvm/test/Analysis/CostModel/RISCV/rvv-vectorinsert.ll +++ b/llvm/test/Analysis/CostModel/RISCV/rvv-vectorinsert.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv64 -mattr=+v < %s | FileCheck %s +; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-min=128 -riscv-v-vector-bits-max=128 < %s | FileCheck %s --check-prefix=RTH-MINMAX ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv64 -mattr=+v -cost-kind=code-size < %s | FileCheck %s --check-prefix=SIZE define void @vector_insert_nxv128i8_0(<vscale x 128 x i8> %v) { @@ -20,6 +21,23 @@ define void @vector_insert_nxv128i8_0(<vscale x 128 x i8> %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_m8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v128i8(<vscale x 128 x i8> %v, <128 x i8> undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; RTH-MINMAX-LABEL: 'vector_insert_nxv128i8_0' +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv1i8(<vscale x 128 x i8> %v, <vscale x 1 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv2i8(<vscale x 128 x i8> %v, <vscale x 2 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv4i8(<vscale x 128 x i8> %v, <vscale x 4 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m1 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv8i8(<vscale x 128 x i8> %v, <vscale x 8 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv16i8(<vscale x 128 x i8> %v, <vscale x 16 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv32i8(<vscale x 128 x i8> %v, <vscale x 32 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv64i8(<vscale x 128 x i8> %v, <vscale x 64 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_mf8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v2i8(<vscale x 128 x i8> %v, <2 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_mf4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v4i8(<vscale x 128 x i8> %v, <4 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_mf2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v8i8(<vscale x 128 x i8> %v, <8 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m1 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v16i8(<vscale x 128 x i8> %v, <16 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v32i8(<vscale x 128 x i8> %v, <32 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v64i8(<vscale x 128 x i8> %v, <64 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v128i8(<vscale x 128 x i8> %v, <128 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; SIZE-LABEL: 'vector_insert_nxv128i8_0' ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv1i8(<vscale x 128 x i8> %v, <vscale x 1 x i8> undef, i64 0) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv2i8(<vscale x 128 x i8> %v, <vscale x 2 x i8> undef, i64 0) @@ -73,6 +91,23 @@ define void @vector_insert_nxv128i8_undef_0() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_m8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v128i8(<vscale x 128 x i8> undef, <128 x i8> undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; RTH-MINMAX-LABEL: 'vector_insert_nxv128i8_undef_0' +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv1i8(<vscale x 128 x i8> undef, <vscale x 1 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv2i8(<vscale x 128 x i8> undef, <vscale x 2 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv4i8(<vscale x 128 x i8> undef, <vscale x 4 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m1 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv8i8(<vscale x 128 x i8> undef, <vscale x 8 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv16i8(<vscale x 128 x i8> undef, <vscale x 16 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv32i8(<vscale x 128 x i8> undef, <vscale x 32 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv64i8(<vscale x 128 x i8> undef, <vscale x 64 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_mf8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v2i8(<vscale x 128 x i8> undef, <2 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_mf4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v4i8(<vscale x 128 x i8> undef, <4 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_mf2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v8i8(<vscale x 128 x i8> undef, <8 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m1 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v16i8(<vscale x 128 x i8> undef, <16 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v32i8(<vscale x 128 x i8> undef, <32 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v64i8(<vscale x 128 x i8> undef, <64 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v128i8(<vscale x 128 x i8> undef, <128 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; SIZE-LABEL: 'vector_insert_nxv128i8_undef_0' ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv1i8(<vscale x 128 x i8> undef, <vscale x 1 x i8> undef, i64 0) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv2i8(<vscale x 128 x i8> undef, <vscale x 2 x i8> undef, i64 0) @@ -126,6 +161,23 @@ define void @vector_insert_nxv128i8_1(<vscale x 128 x i8> %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_m8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v128i8(<vscale x 128 x i8> %v, <128 x i8> undef, i64 128) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; RTH-MINMAX-LABEL: 'vector_insert_nxv128i8_1' +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv1i8(<vscale x 128 x i8> %v, <vscale x 1 x i8> undef, i64 1) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv2i8(<vscale x 128 x i8> %v, <vscale x 2 x i8> undef, i64 2) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv4i8(<vscale x 128 x i8> %v, <vscale x 4 x i8> undef, i64 4) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m1 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv8i8(<vscale x 128 x i8> %v, <vscale x 8 x i8> undef, i64 8) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv16i8(<vscale x 128 x i8> %v, <vscale x 16 x i8> undef, i64 16) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv32i8(<vscale x 128 x i8> %v, <vscale x 32 x i8> undef, i64 32) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv64i8(<vscale x 128 x i8> %v, <vscale x 64 x i8> undef, i64 64) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_mf8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v2i8(<vscale x 128 x i8> %v, <2 x i8> undef, i64 2) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_mf4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v4i8(<vscale x 128 x i8> %v, <4 x i8> undef, i64 4) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_mf2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v8i8(<vscale x 128 x i8> %v, <8 x i8> undef, i64 8) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_m1 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v16i8(<vscale x 128 x i8> %v, <16 x i8> undef, i64 16) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_m2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v32i8(<vscale x 128 x i8> %v, <32 x i8> undef, i64 32) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_m4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v64i8(<vscale x 128 x i8> %v, <64 x i8> undef, i64 64) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_m8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v128i8(<vscale x 128 x i8> %v, <128 x i8> undef, i64 128) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; SIZE-LABEL: 'vector_insert_nxv128i8_1' ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv1i8(<vscale x 128 x i8> %v, <vscale x 1 x i8> undef, i64 1) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv2i8(<vscale x 128 x i8> %v, <vscale x 2 x i8> undef, i64 2) @@ -179,6 +231,23 @@ define void @vector_insert_nxv128i8_undef_1() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_m8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v128i8(<vscale x 128 x i8> undef, <128 x i8> undef, i64 128) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; RTH-MINMAX-LABEL: 'vector_insert_nxv128i8_undef_1' +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv1i8(<vscale x 128 x i8> undef, <vscale x 1 x i8> undef, i64 1) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv2i8(<vscale x 128 x i8> undef, <vscale x 2 x i8> undef, i64 2) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv4i8(<vscale x 128 x i8> undef, <vscale x 4 x i8> undef, i64 4) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m1 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv8i8(<vscale x 128 x i8> undef, <vscale x 8 x i8> undef, i64 8) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv16i8(<vscale x 128 x i8> undef, <vscale x 16 x i8> undef, i64 16) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv32i8(<vscale x 128 x i8> undef, <vscale x 32 x i8> undef, i64 32) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_m8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv64i8(<vscale x 128 x i8> undef, <vscale x 64 x i8> undef, i64 64) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_mf8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v2i8(<vscale x 128 x i8> undef, <2 x i8> undef, i64 2) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_mf4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v4i8(<vscale x 128 x i8> undef, <4 x i8> undef, i64 4) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_mf2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v8i8(<vscale x 128 x i8> undef, <8 x i8> undef, i64 8) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_m1 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v16i8(<vscale x 128 x i8> undef, <16 x i8> undef, i64 16) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_m2 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v32i8(<vscale x 128 x i8> undef, <32 x i8> undef, i64 32) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_m4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v64i8(<vscale x 128 x i8> undef, <64 x i8> undef, i64 64) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fixed_m8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.v128i8(<vscale x 128 x i8> undef, <128 x i8> undef, i64 128) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; SIZE-LABEL: 'vector_insert_nxv128i8_undef_1' ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf8 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv1i8(<vscale x 128 x i8> undef, <vscale x 1 x i8> undef, i64 1) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %scalable_mf4 = call <vscale x 128 x i8> @llvm.vector.insert.nxv128i8.nxv2i8(<vscale x 128 x i8> undef, <vscale x 2 x i8> undef, i64 2) @@ -219,20 +288,30 @@ define void @vector_insert_v128i8_0(<128 x i8> %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf8 = call <128 x i8> @llvm.vector.insert.v128i8.v2i8(<128 x i8> %v, <2 x i8> undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf4 = call <128 x i8> @llvm.vector.insert.v128i8.v4i8(<128 x i8> %v, <4 x i8> undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf2 = call <128 x i8> @llvm.vector.insert.v128i8.v8i8(<128 x i8> %v, <8 x i8> undef, i64 0) -; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m1 = call <128 x i8> @llvm.vector.insert.v128i8.v16i8(<128 x i8> %v, <16 x i8> undef, i64 0) -; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m2 = call <128 x i8> @llvm.vector.insert.v128i8.v32i8(<128 x i8> %v, <32 x i8> undef, i64 0) -; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m4 = call <128 x i8> @llvm.vector.insert.v128i8.v64i8(<128 x i8> %v, <64 x i8> undef, i64 0) -; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m8 = call <128 x i8> @llvm.vector.insert.v128i8.v128i8(<128 x i8> %v, <128 x i8> undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m1 = call <128 x i8> @llvm.vector.insert.v128i8.v16i8(<128 x i8> %v, <16 x i8> undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m2 = call <128 x i8> @llvm.vector.insert.v128i8.v32i8(<128 x i8> %v, <32 x i8> undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m4 = call <128 x i8> @llvm.vector.insert.v128i8.v64i8(<128 x i8> %v, <64 x i8> undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m8 = call <128 x i8> @llvm.vector.insert.v128i8.v128i8(<128 x i8> %v, <128 x i8> undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; RTH-MINMAX-LABEL: 'vector_insert_v128i8_0' +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf8 = call <128 x i8> @llvm.vector.insert.v128i8.v2i8(<128 x i8> %v, <2 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf4 = call <128 x i8> @llvm.vector.insert.v128i8.v4i8(<128 x i8> %v, <4 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf2 = call <128 x i8> @llvm.vector.insert.v128i8.v8i8(<128 x i8> %v, <8 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m1 = call <128 x i8> @llvm.vector.insert.v128i8.v16i8(<128 x i8> %v, <16 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m2 = call <128 x i8> @llvm.vector.insert.v128i8.v32i8(<128 x i8> %v, <32 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m4 = call <128 x i8> @llvm.vector.insert.v128i8.v64i8(<128 x i8> %v, <64 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m8 = call <128 x i8> @llvm.vector.insert.v128i8.v128i8(<128 x i8> %v, <128 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; SIZE-LABEL: 'vector_insert_v128i8_0' ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_mf8 = call <128 x i8> @llvm.vector.insert.v128i8.v2i8(<128 x i8> %v, <2 x i8> undef, i64 0) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_mf4 = call <128 x i8> @llvm.vector.insert.v128i8.v4i8(<128 x i8> %v, <4 x i8> undef, i64 0) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_mf2 = call <128 x i8> @llvm.vector.insert.v128i8.v8i8(<128 x i8> %v, <8 x i8> undef, i64 0) -; SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m1 = call <128 x i8> @llvm.vector.insert.v128i8.v16i8(<128 x i8> %v, <16 x i8> undef, i64 0) -; SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m2 = call <128 x i8> @llvm.vector.insert.v128i8.v32i8(<128 x i8> %v, <32 x i8> undef, i64 0) -; SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m4 = call <128 x i8> @llvm.vector.insert.v128i8.v64i8(<128 x i8> %v, <64 x i8> undef, i64 0) -; SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m8 = call <128 x i8> @llvm.vector.insert.v128i8.v128i8(<128 x i8> %v, <128 x i8> undef, i64 0) +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_m1 = call <128 x i8> @llvm.vector.insert.v128i8.v16i8(<128 x i8> %v, <16 x i8> undef, i64 0) +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_m2 = call <128 x i8> @llvm.vector.insert.v128i8.v32i8(<128 x i8> %v, <32 x i8> undef, i64 0) +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_m4 = call <128 x i8> @llvm.vector.insert.v128i8.v64i8(<128 x i8> %v, <64 x i8> undef, i64 0) +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_m8 = call <128 x i8> @llvm.vector.insert.v128i8.v128i8(<128 x i8> %v, <128 x i8> undef, i64 0) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %fixed_mf8 = call <128 x i8> @llvm.vector.insert.v128i8.v2i8(<128 x i8> %v, <2 x i8> undef, i64 0) @@ -250,20 +329,30 @@ define void @vector_insert_v128i8_undef_0() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf8 = call <128 x i8> @llvm.vector.insert.v128i8.v2i8(<128 x i8> undef, <2 x i8> undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf4 = call <128 x i8> @llvm.vector.insert.v128i8.v4i8(<128 x i8> undef, <4 x i8> undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf2 = call <128 x i8> @llvm.vector.insert.v128i8.v8i8(<128 x i8> undef, <8 x i8> undef, i64 0) -; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m1 = call <128 x i8> @llvm.vector.insert.v128i8.v16i8(<128 x i8> undef, <16 x i8> undef, i64 0) -; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m2 = call <128 x i8> @llvm.vector.insert.v128i8.v32i8(<128 x i8> undef, <32 x i8> undef, i64 0) -; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m4 = call <128 x i8> @llvm.vector.insert.v128i8.v64i8(<128 x i8> undef, <64 x i8> undef, i64 0) -; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m8 = call <128 x i8> @llvm.vector.insert.v128i8.v128i8(<128 x i8> undef, <128 x i8> undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m1 = call <128 x i8> @llvm.vector.insert.v128i8.v16i8(<128 x i8> undef, <16 x i8> undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m2 = call <128 x i8> @llvm.vector.insert.v128i8.v32i8(<128 x i8> undef, <32 x i8> undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m4 = call <128 x i8> @llvm.vector.insert.v128i8.v64i8(<128 x i8> undef, <64 x i8> undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m8 = call <128 x i8> @llvm.vector.insert.v128i8.v128i8(<128 x i8> undef, <128 x i8> undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; RTH-MINMAX-LABEL: 'vector_insert_v128i8_undef_0' +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf8 = call <128 x i8> @llvm.vector.insert.v128i8.v2i8(<128 x i8> undef, <2 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf4 = call <128 x i8> @llvm.vector.insert.v128i8.v4i8(<128 x i8> undef, <4 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf2 = call <128 x i8> @llvm.vector.insert.v128i8.v8i8(<128 x i8> undef, <8 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m1 = call <128 x i8> @llvm.vector.insert.v128i8.v16i8(<128 x i8> undef, <16 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m2 = call <128 x i8> @llvm.vector.insert.v128i8.v32i8(<128 x i8> undef, <32 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m4 = call <128 x i8> @llvm.vector.insert.v128i8.v64i8(<128 x i8> undef, <64 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m8 = call <128 x i8> @llvm.vector.insert.v128i8.v128i8(<128 x i8> undef, <128 x i8> undef, i64 0) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; SIZE-LABEL: 'vector_insert_v128i8_undef_0' ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_mf8 = call <128 x i8> @llvm.vector.insert.v128i8.v2i8(<128 x i8> undef, <2 x i8> undef, i64 0) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_mf4 = call <128 x i8> @llvm.vector.insert.v128i8.v4i8(<128 x i8> undef, <4 x i8> undef, i64 0) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_mf2 = call <128 x i8> @llvm.vector.insert.v128i8.v8i8(<128 x i8> undef, <8 x i8> undef, i64 0) -; SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m1 = call <128 x i8> @llvm.vector.insert.v128i8.v16i8(<128 x i8> undef, <16 x i8> undef, i64 0) -; SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m2 = call <128 x i8> @llvm.vector.insert.v128i8.v32i8(<128 x i8> undef, <32 x i8> undef, i64 0) -; SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m4 = call <128 x i8> @llvm.vector.insert.v128i8.v64i8(<128 x i8> undef, <64 x i8> undef, i64 0) -; SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %fixed_m8 = call <128 x i8> @llvm.vector.insert.v128i8.v128i8(<128 x i8> undef, <128 x i8> undef, i64 0) +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_m1 = call <128 x i8> @llvm.vector.insert.v128i8.v16i8(<128 x i8> undef, <16 x i8> undef, i64 0) +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_m2 = call <128 x i8> @llvm.vector.insert.v128i8.v32i8(<128 x i8> undef, <32 x i8> undef, i64 0) +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_m4 = call <128 x i8> @llvm.vector.insert.v128i8.v64i8(<128 x i8> undef, <64 x i8> undef, i64 0) +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_m8 = call <128 x i8> @llvm.vector.insert.v128i8.v128i8(<128 x i8> undef, <128 x i8> undef, i64 0) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %fixed_mf8 = call <128 x i8> @llvm.vector.insert.v128i8.v2i8(<128 x i8> undef, <2 x i8> undef, i64 0) @@ -286,6 +375,15 @@ define void @vector_insert_v128i8_1(<128 x i8> %v) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m4 = call <128 x i8> @llvm.vector.insert.v128i8.v64i8(<128 x i8> %v, <64 x i8> undef, i64 64) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; RTH-MINMAX-LABEL: 'vector_insert_v128i8_1' +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf8 = call <128 x i8> @llvm.vector.insert.v128i8.v2i8(<128 x i8> %v, <2 x i8> undef, i64 2) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf4 = call <128 x i8> @llvm.vector.insert.v128i8.v4i8(<128 x i8> %v, <4 x i8> undef, i64 4) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf2 = call <128 x i8> @llvm.vector.insert.v128i8.v8i8(<128 x i8> %v, <8 x i8> undef, i64 8) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m1 = call <128 x i8> @llvm.vector.insert.v128i8.v16i8(<128 x i8> %v, <16 x i8> undef, i64 16) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m2 = call <128 x i8> @llvm.vector.insert.v128i8.v32i8(<128 x i8> %v, <32 x i8> undef, i64 32) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m4 = call <128 x i8> @llvm.vector.insert.v128i8.v64i8(<128 x i8> %v, <64 x i8> undef, i64 64) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; SIZE-LABEL: 'vector_insert_v128i8_1' ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_mf8 = call <128 x i8> @llvm.vector.insert.v128i8.v2i8(<128 x i8> %v, <2 x i8> undef, i64 2) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_mf4 = call <128 x i8> @llvm.vector.insert.v128i8.v4i8(<128 x i8> %v, <4 x i8> undef, i64 4) @@ -315,6 +413,15 @@ define void @vector_insert_v128i8_undef_1() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m4 = call <128 x i8> @llvm.vector.insert.v128i8.v64i8(<128 x i8> undef, <64 x i8> undef, i64 64) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; RTH-MINMAX-LABEL: 'vector_insert_v128i8_undef_1' +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf8 = call <128 x i8> @llvm.vector.insert.v128i8.v2i8(<128 x i8> undef, <2 x i8> undef, i64 2) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf4 = call <128 x i8> @llvm.vector.insert.v128i8.v4i8(<128 x i8> undef, <4 x i8> undef, i64 4) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_mf2 = call <128 x i8> @llvm.vector.insert.v128i8.v8i8(<128 x i8> undef, <8 x i8> undef, i64 8) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m1 = call <128 x i8> @llvm.vector.insert.v128i8.v16i8(<128 x i8> undef, <16 x i8> undef, i64 16) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m2 = call <128 x i8> @llvm.vector.insert.v128i8.v32i8(<128 x i8> undef, <32 x i8> undef, i64 32) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %fixed_m4 = call <128 x i8> @llvm.vector.insert.v128i8.v64i8(<128 x i8> undef, <64 x i8> undef, i64 64) +; RTH-MINMAX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; SIZE-LABEL: 'vector_insert_v128i8_undef_1' ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_mf8 = call <128 x i8> @llvm.vector.insert.v128i8.v2i8(<128 x i8> undef, <2 x i8> undef, i64 2) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %fixed_mf4 = call <128 x i8> @llvm.vector.insert.v128i8.v4i8(<128 x i8> undef, <4 x i8> undef, i64 4) diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll index fc4a6b1..46bf315 100644 --- a/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll +++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll @@ -45,9 +45,9 @@ define void @broadcast_scalable() #0{ ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %38 = shufflevector <vscale x 2 x i1> undef, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %39 = shufflevector <vscale x 4 x i1> undef, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer ; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %40 = shufflevector <vscale x 8 x i1> undef, <vscale x 8 x i1> undef, <vscale x 8 x i32> zeroinitializer -; CHECK-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %41 = shufflevector <vscale x 16 x i1> undef, <vscale x 16 x i1> undef, <vscale x 16 x i32> zeroinitializer -; CHECK-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %42 = shufflevector <vscale x 32 x i1> undef, <vscale x 32 x i1> undef, <vscale x 32 x i32> zeroinitializer -; CHECK-NEXT: Cost Model: Found an estimated cost of 41 for instruction: %43 = shufflevector <vscale x 64 x i1> undef, <vscale x 64 x i1> undef, <vscale x 64 x i32> zeroinitializer +; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %41 = shufflevector <vscale x 16 x i1> undef, <vscale x 16 x i1> undef, <vscale x 16 x i32> zeroinitializer +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %42 = shufflevector <vscale x 32 x i1> undef, <vscale x 32 x i1> undef, <vscale x 32 x i32> zeroinitializer +; CHECK-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %43 = shufflevector <vscale x 64 x i1> undef, <vscale x 64 x i1> undef, <vscale x 64 x i32> zeroinitializer ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; SIZE-LABEL: 'broadcast_scalable' @@ -92,9 +92,9 @@ define void @broadcast_scalable() #0{ ; SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %38 = shufflevector <vscale x 2 x i1> undef, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer ; SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %39 = shufflevector <vscale x 4 x i1> undef, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer ; SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %40 = shufflevector <vscale x 8 x i1> undef, <vscale x 8 x i1> undef, <vscale x 8 x i32> zeroinitializer -; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %41 = shufflevector <vscale x 16 x i1> undef, <vscale x 16 x i1> undef, <vscale x 16 x i32> zeroinitializer -; SIZE-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %42 = shufflevector <vscale x 32 x i1> undef, <vscale x 32 x i1> undef, <vscale x 32 x i32> zeroinitializer -; SIZE-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %43 = shufflevector <vscale x 64 x i1> undef, <vscale x 64 x i1> undef, <vscale x 64 x i32> zeroinitializer +; SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %41 = shufflevector <vscale x 16 x i1> undef, <vscale x 16 x i1> undef, <vscale x 16 x i32> zeroinitializer +; SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %42 = shufflevector <vscale x 32 x i1> undef, <vscale x 32 x i1> undef, <vscale x 32 x i32> zeroinitializer +; SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %43 = shufflevector <vscale x 64 x i1> undef, <vscale x 64 x i1> undef, <vscale x 64 x i32> zeroinitializer ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %zero = shufflevector <vscale x 1 x half> undef, <vscale x 1 x half> undef, <vscale x 1 x i32> zeroinitializer diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll index 19a571b..9a333dc 100644 --- a/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll +++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll @@ -1,6 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py ; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv32 -mattr=+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-min=-1 | FileCheck %s -; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv32 -mattr=+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-min=128 -riscv-v-vector-bits-max=128 | FileCheck %s ; RUN: opt < %s -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=riscv32 -mattr=+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-min=-1 | FileCheck %s --check-prefix=SIZE ; Check that we don't crash querying costs when vectors are not enabled. ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv32 diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-reverse.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-reverse.ll index 146909c..e80dbe3 100644 --- a/llvm/test/Analysis/CostModel/RISCV/shuffle-reverse.ll +++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-reverse.ll @@ -20,21 +20,21 @@ define void @reverse() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2i16 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> <i32 1, i32 0> ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i16 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0> ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0> ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0> -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2i64 = shufflevector <2 x i64> undef, <2 x i64> undef, <2 x i32> <i32 1, i32 0> -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0> ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = shufflevector <2 x half> undef, <2 x half> undef, <2 x i32> <i32 1, i32 0> ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0> ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f32 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> <i32 1, i32 0> ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f32 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0> -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f64 = shufflevector <2 x double> undef, <2 x double> undef, <2 x i32> <i32 1, i32 0> -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0> ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; SIZE-LABEL: 'reverse' diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-2-icmps-of-0-and-or.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-2-icmps-of-0-and-or.mir new file mode 100644 index 0000000..2ce5c69 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-2-icmps-of-0-and-or.mir @@ -0,0 +1,1244 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s +# REQUIRES: asserts + + +--- +name: valid_and_eq_0_eq_0_s32 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $w0, $w1 + + ; CHECK-LABEL: name: valid_and_eq_0_eq_0_s32 + ; CHECK: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s32) = COPY $w0 + ; CHECK-NEXT: %y:_(s32) = COPY $w1 + ; CHECK-NEXT: %zero:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR %x, %y + ; CHECK-NEXT: %and:_(s1) = G_ICMP intpred(eq), [[OR]](s32), %zero + ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %and(s1) + ; CHECK-NEXT: $w0 = COPY %zext(s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %x:_(s32) = COPY $w0 + %y:_(s32) = COPY $w1 + %zero:_(s32) = G_CONSTANT i32 0 + %cmp1:_(s1) = G_ICMP intpred(eq), %x:_(s32), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(eq), %y:_(s32), %zero:_ + %and:_(s1) = G_AND %cmp1, %cmp2 + %zext:_(s32) = G_ZEXT %and:_(s1) + $w0 = COPY %zext + RET_ReallyLR implicit $w0 + +... +--- +name: invalid_and_eq_1_eq_0_s32 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $w0, $w1 + + ; CHECK-LABEL: name: invalid_and_eq_1_eq_0_s32 + ; CHECK: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s32) = COPY $w0 + ; CHECK-NEXT: %y:_(s32) = COPY $w1 + ; CHECK-NEXT: %zero:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %one:_(s32) = G_CONSTANT i32 1 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(eq), %x(s32), %one + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(eq), %y(s32), %zero + ; CHECK-NEXT: %and:_(s1) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %and(s1) + ; CHECK-NEXT: $w0 = COPY %zext(s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %x:_(s32) = COPY $w0 + %y:_(s32) = COPY $w1 + %zero:_(s32) = G_CONSTANT i32 0 + %one:_(s32) = G_CONSTANT i32 1 + %cmp1:_(s1) = G_ICMP intpred(eq), %x:_(s32), %one:_ + %cmp2:_(s1) = G_ICMP intpred(eq), %y:_(s32), %zero:_ + %and:_(s1) = G_AND %cmp1, %cmp2 + %zext:_(s32) = G_ZEXT %and:_(s1) + $w0 = COPY %zext + RET_ReallyLR implicit $w0 + +... +--- +name: invalid_and_eq_0_eq_1_s32 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $w0, $w1 + + ; CHECK-LABEL: name: invalid_and_eq_0_eq_1_s32 + ; CHECK: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s32) = COPY $w0 + ; CHECK-NEXT: %y:_(s32) = COPY $w1 + ; CHECK-NEXT: %zero:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %one:_(s32) = G_CONSTANT i32 1 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(eq), %x(s32), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(eq), %y(s32), %one + ; CHECK-NEXT: %and:_(s1) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %and(s1) + ; CHECK-NEXT: $w0 = COPY %zext(s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %x:_(s32) = COPY $w0 + %y:_(s32) = COPY $w1 + %zero:_(s32) = G_CONSTANT i32 0 + %one:_(s32) = G_CONSTANT i32 1 + %cmp1:_(s1) = G_ICMP intpred(eq), %x:_(s32), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(eq), %y:_(s32), %one:_ + %and:_(s1) = G_AND %cmp1, %cmp2 + %zext:_(s32) = G_ZEXT %and:_(s1) + $w0 = COPY %zext + RET_ReallyLR implicit $w0 + +... +--- +name: invalid_and_ne_0_eq_0_s32 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $w0, $w1 + + ; CHECK-LABEL: name: invalid_and_ne_0_eq_0_s32 + ; CHECK: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s32) = COPY $w0 + ; CHECK-NEXT: %y:_(s32) = COPY $w1 + ; CHECK-NEXT: %zero:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(ne), %x(s32), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(eq), %y(s32), %zero + ; CHECK-NEXT: %and:_(s1) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %and(s1) + ; CHECK-NEXT: $w0 = COPY %zext(s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %x:_(s32) = COPY $w0 + %y:_(s32) = COPY $w1 + %zero:_(s32) = G_CONSTANT i32 0 + %cmp1:_(s1) = G_ICMP intpred(ne), %x:_(s32), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(eq), %y:_(s32), %zero:_ + %and:_(s1) = G_AND %cmp1, %cmp2 + %zext:_(s32) = G_ZEXT %and:_(s1) + $w0 = COPY %zext + RET_ReallyLR implicit $w0 + +... +--- +name: invalid_and_eq_0_ne_0_s32 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $w0, $w1 + + ; CHECK-LABEL: name: invalid_and_eq_0_ne_0_s32 + ; CHECK: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s32) = COPY $w0 + ; CHECK-NEXT: %y:_(s32) = COPY $w1 + ; CHECK-NEXT: %zero:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(eq), %x(s32), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(ne), %y(s32), %zero + ; CHECK-NEXT: %and:_(s1) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %and(s1) + ; CHECK-NEXT: $w0 = COPY %zext(s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %x:_(s32) = COPY $w0 + %y:_(s32) = COPY $w1 + %zero:_(s32) = G_CONSTANT i32 0 + %cmp1:_(s1) = G_ICMP intpred(eq), %x:_(s32), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(ne), %y:_(s32), %zero:_ + %and:_(s1) = G_AND %cmp1, %cmp2 + %zext:_(s32) = G_ZEXT %and:_(s1) + $w0 = COPY %zext + RET_ReallyLR implicit $w0 + +... +--- +name: invalid_and_ne_0_ne_0_s32 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $w0, $w1 + + ; CHECK-LABEL: name: invalid_and_ne_0_ne_0_s32 + ; CHECK: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s32) = COPY $w0 + ; CHECK-NEXT: %y:_(s32) = COPY $w1 + ; CHECK-NEXT: %zero:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(ne), %x(s32), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(ne), %y(s32), %zero + ; CHECK-NEXT: %and:_(s1) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %and(s1) + ; CHECK-NEXT: $w0 = COPY %zext(s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %x:_(s32) = COPY $w0 + %y:_(s32) = COPY $w1 + %zero:_(s32) = G_CONSTANT i32 0 + %cmp1:_(s1) = G_ICMP intpred(ne), %x:_(s32), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(ne), %y:_(s32), %zero:_ + %and:_(s1) = G_AND %cmp1, %cmp2 + %zext:_(s32) = G_ZEXT %and:_(s1) + $w0 = COPY %zext + RET_ReallyLR implicit $w0 + +... +--- +name: valid_or_ne_0_ne_0_s32 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $w0, $w1 + + ; CHECK-LABEL: name: valid_or_ne_0_ne_0_s32 + ; CHECK: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s32) = COPY $w0 + ; CHECK-NEXT: %y:_(s32) = COPY $w1 + ; CHECK-NEXT: %zero:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR %x, %y + ; CHECK-NEXT: %or:_(s1) = G_ICMP intpred(ne), [[OR]](s32), %zero + ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %or(s1) + ; CHECK-NEXT: $w0 = COPY %zext(s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %x:_(s32) = COPY $w0 + %y:_(s32) = COPY $w1 + %zero:_(s32) = G_CONSTANT i32 0 + %cmp1:_(s1) = G_ICMP intpred(ne), %x:_(s32), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(ne), %y:_(s32), %zero:_ + %or:_(s1) = G_OR %cmp1, %cmp2 + %zext:_(s32) = G_ZEXT %or:_(s1) + $w0 = COPY %zext + RET_ReallyLR implicit $w0 + +... +--- +name: invalid_or_ne_1_ne_0_s32 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $w0, $w1 + + ; CHECK-LABEL: name: invalid_or_ne_1_ne_0_s32 + ; CHECK: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s32) = COPY $w0 + ; CHECK-NEXT: %y:_(s32) = COPY $w1 + ; CHECK-NEXT: %zero:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %one:_(s32) = G_CONSTANT i32 1 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(ne), %x(s32), %one + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(ne), %y(s32), %zero + ; CHECK-NEXT: %or:_(s1) = G_OR %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %or(s1) + ; CHECK-NEXT: $w0 = COPY %zext(s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %x:_(s32) = COPY $w0 + %y:_(s32) = COPY $w1 + %zero:_(s32) = G_CONSTANT i32 0 + %one:_(s32) = G_CONSTANT i32 1 + %cmp1:_(s1) = G_ICMP intpred(ne), %x:_(s32), %one:_ + %cmp2:_(s1) = G_ICMP intpred(ne), %y:_(s32), %zero:_ + %or:_(s1) = G_OR %cmp1, %cmp2 + %zext:_(s32) = G_ZEXT %or:_(s1) + $w0 = COPY %zext + RET_ReallyLR implicit $w0 + +... +--- +name: invalid_or_ne_0_ne_1_s32 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $w0, $w1 + + ; CHECK-LABEL: name: invalid_or_ne_0_ne_1_s32 + ; CHECK: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s32) = COPY $w0 + ; CHECK-NEXT: %y:_(s32) = COPY $w1 + ; CHECK-NEXT: %zero:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %one:_(s32) = G_CONSTANT i32 1 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(ne), %x(s32), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(ne), %y(s32), %one + ; CHECK-NEXT: %or:_(s1) = G_OR %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %or(s1) + ; CHECK-NEXT: $w0 = COPY %zext(s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %x:_(s32) = COPY $w0 + %y:_(s32) = COPY $w1 + %zero:_(s32) = G_CONSTANT i32 0 + %one:_(s32) = G_CONSTANT i32 1 + %cmp1:_(s1) = G_ICMP intpred(ne), %x:_(s32), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(ne), %y:_(s32), %one:_ + %or:_(s1) = G_OR %cmp1, %cmp2 + %zext:_(s32) = G_ZEXT %or:_(s1) + $w0 = COPY %zext + RET_ReallyLR implicit $w0 + +... +--- +name: invalid_or_eq_0_ne_0_s32 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $w0, $w1 + + ; CHECK-LABEL: name: invalid_or_eq_0_ne_0_s32 + ; CHECK: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s32) = COPY $w0 + ; CHECK-NEXT: %y:_(s32) = COPY $w1 + ; CHECK-NEXT: %zero:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(eq), %x(s32), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(ne), %y(s32), %zero + ; CHECK-NEXT: %or:_(s1) = G_OR %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %or(s1) + ; CHECK-NEXT: $w0 = COPY %zext(s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %x:_(s32) = COPY $w0 + %y:_(s32) = COPY $w1 + %zero:_(s32) = G_CONSTANT i32 0 + %cmp1:_(s1) = G_ICMP intpred(eq), %x:_(s32), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(ne), %y:_(s32), %zero:_ + %or:_(s1) = G_OR %cmp1, %cmp2 + %zext:_(s32) = G_ZEXT %or:_(s1) + $w0 = COPY %zext + RET_ReallyLR implicit $w0 + +... +--- +name: invalid_or_ne_0_eq_0_s32 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $w0, $w1 + + ; CHECK-LABEL: name: invalid_or_ne_0_eq_0_s32 + ; CHECK: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s32) = COPY $w0 + ; CHECK-NEXT: %y:_(s32) = COPY $w1 + ; CHECK-NEXT: %zero:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(ne), %x(s32), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(eq), %y(s32), %zero + ; CHECK-NEXT: %or:_(s1) = G_OR %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s32) = G_ZEXT %or(s1) + ; CHECK-NEXT: $w0 = COPY %zext(s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %x:_(s32) = COPY $w0 + %y:_(s32) = COPY $w1 + %zero:_(s32) = G_CONSTANT i32 0 + %cmp1:_(s1) = G_ICMP intpred(ne), %x:_(s32), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(eq), %y:_(s32), %zero:_ + %or:_(s1) = G_OR %cmp1, %cmp2 + %zext:_(s32) = G_ZEXT %or:_(s1) + $w0 = COPY %zext + RET_ReallyLR implicit $w0 + +... + +--- +name: valid_and_eq_0_eq_0_s64 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: valid_and_eq_0_eq_0_s64 + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s64) = COPY $x0 + ; CHECK-NEXT: %y:_(s64) = COPY $x1 + ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR %x, %y + ; CHECK-NEXT: %and:_(s1) = G_ICMP intpred(eq), [[OR]](s64), %zero + ; CHECK-NEXT: %zext:_(s64) = G_ZEXT %and(s1) + ; CHECK-NEXT: $x0 = COPY %zext(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(s64) = COPY $x0 + %y:_(s64) = COPY $x1 + %zero:_(s64) = G_CONSTANT i64 0 + %cmp1:_(s1) = G_ICMP intpred(eq), %x:_(s64), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(eq), %y:_(s64), %zero:_ + %and:_(s1) = G_AND %cmp1, %cmp2 + %zext:_(s64) = G_ZEXT %and:_(s1) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_and_eq_1_eq_0_s64 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_and_eq_1_eq_0_s64 + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s64) = COPY $x0 + ; CHECK-NEXT: %y:_(s64) = COPY $x1 + ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: %one:_(s64) = G_CONSTANT i64 1 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(eq), %x(s64), %one + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(eq), %y(s64), %zero + ; CHECK-NEXT: %and:_(s1) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s64) = G_ZEXT %and(s1) + ; CHECK-NEXT: $x0 = COPY %zext(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(s64) = COPY $x0 + %y:_(s64) = COPY $x1 + %zero:_(s64) = G_CONSTANT i64 0 + %one:_(s64) = G_CONSTANT i64 1 + %cmp1:_(s1) = G_ICMP intpred(eq), %x:_(s64), %one:_ + %cmp2:_(s1) = G_ICMP intpred(eq), %y:_(s64), %zero:_ + %and:_(s1) = G_AND %cmp1, %cmp2 + %zext:_(s64) = G_ZEXT %and:_(s1) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_and_eq_0_eq_1_s64 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_and_eq_0_eq_1_s64 + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s64) = COPY $x0 + ; CHECK-NEXT: %y:_(s64) = COPY $x1 + ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: %one:_(s64) = G_CONSTANT i64 1 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(eq), %x(s64), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(eq), %y(s64), %one + ; CHECK-NEXT: %and:_(s1) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s64) = G_ZEXT %and(s1) + ; CHECK-NEXT: $x0 = COPY %zext(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(s64) = COPY $x0 + %y:_(s64) = COPY $x1 + %zero:_(s64) = G_CONSTANT i64 0 + %one:_(s64) = G_CONSTANT i64 1 + %cmp1:_(s1) = G_ICMP intpred(eq), %x:_(s64), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(eq), %y:_(s64), %one:_ + %and:_(s1) = G_AND %cmp1, %cmp2 + %zext:_(s64) = G_ZEXT %and:_(s1) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_and_ne_0_eq_0_s64 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_and_ne_0_eq_0_s64 + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s64) = COPY $x0 + ; CHECK-NEXT: %y:_(s64) = COPY $x1 + ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(ne), %x(s64), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(eq), %y(s64), %zero + ; CHECK-NEXT: %and:_(s1) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s64) = G_ZEXT %and(s1) + ; CHECK-NEXT: $x0 = COPY %zext(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(s64) = COPY $x0 + %y:_(s64) = COPY $x1 + %zero:_(s64) = G_CONSTANT i64 0 + %cmp1:_(s1) = G_ICMP intpred(ne), %x:_(s64), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(eq), %y:_(s64), %zero:_ + %and:_(s1) = G_AND %cmp1, %cmp2 + %zext:_(s64) = G_ZEXT %and:_(s1) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_and_eq_0_ne_0_s64 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_and_eq_0_ne_0_s64 + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s64) = COPY $x0 + ; CHECK-NEXT: %y:_(s64) = COPY $x1 + ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(eq), %x(s64), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(ne), %y(s64), %zero + ; CHECK-NEXT: %and:_(s1) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s64) = G_ZEXT %and(s1) + ; CHECK-NEXT: $x0 = COPY %zext(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(s64) = COPY $x0 + %y:_(s64) = COPY $x1 + %zero:_(s64) = G_CONSTANT i64 0 + %cmp1:_(s1) = G_ICMP intpred(eq), %x:_(s64), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(ne), %y:_(s64), %zero:_ + %and:_(s1) = G_AND %cmp1, %cmp2 + %zext:_(s64) = G_ZEXT %and:_(s1) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_and_ne_0_ne_0_s64 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_and_ne_0_ne_0_s64 + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s64) = COPY $x0 + ; CHECK-NEXT: %y:_(s64) = COPY $x1 + ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(ne), %x(s64), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(ne), %y(s64), %zero + ; CHECK-NEXT: %and:_(s1) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s64) = G_ZEXT %and(s1) + ; CHECK-NEXT: $x0 = COPY %zext(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(s64) = COPY $x0 + %y:_(s64) = COPY $x1 + %zero:_(s64) = G_CONSTANT i64 0 + %cmp1:_(s1) = G_ICMP intpred(ne), %x:_(s64), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(ne), %y:_(s64), %zero:_ + %and:_(s1) = G_AND %cmp1, %cmp2 + %zext:_(s64) = G_ZEXT %and:_(s1) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: valid_or_ne_0_ne_0_s64 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: valid_or_ne_0_ne_0_s64 + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s64) = COPY $x0 + ; CHECK-NEXT: %y:_(s64) = COPY $x1 + ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR %x, %y + ; CHECK-NEXT: %or:_(s1) = G_ICMP intpred(ne), [[OR]](s64), %zero + ; CHECK-NEXT: %zext:_(s64) = G_ZEXT %or(s1) + ; CHECK-NEXT: $x0 = COPY %zext(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(s64) = COPY $x0 + %y:_(s64) = COPY $x1 + %zero:_(s64) = G_CONSTANT i64 0 + %cmp1:_(s1) = G_ICMP intpred(ne), %x:_(s64), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(ne), %y:_(s64), %zero:_ + %or:_(s1) = G_OR %cmp1, %cmp2 + %zext:_(s64) = G_ZEXT %or:_(s1) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_or_ne_1_ne_0_s64 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_or_ne_1_ne_0_s64 + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s64) = COPY $x0 + ; CHECK-NEXT: %y:_(s64) = COPY $x1 + ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: %one:_(s64) = G_CONSTANT i64 1 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(ne), %x(s64), %one + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(ne), %y(s64), %zero + ; CHECK-NEXT: %or:_(s1) = G_OR %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s64) = G_ZEXT %or(s1) + ; CHECK-NEXT: $x0 = COPY %zext(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(s64) = COPY $x0 + %y:_(s64) = COPY $x1 + %zero:_(s64) = G_CONSTANT i64 0 + %one:_(s64) = G_CONSTANT i64 1 + %cmp1:_(s1) = G_ICMP intpred(ne), %x:_(s64), %one:_ + %cmp2:_(s1) = G_ICMP intpred(ne), %y:_(s64), %zero:_ + %or:_(s1) = G_OR %cmp1, %cmp2 + %zext:_(s64) = G_ZEXT %or:_(s1) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_or_ne_0_ne_1_s64 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_or_ne_0_ne_1_s64 + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s64) = COPY $x0 + ; CHECK-NEXT: %y:_(s64) = COPY $x1 + ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: %one:_(s64) = G_CONSTANT i64 1 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(ne), %x(s64), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(ne), %y(s64), %one + ; CHECK-NEXT: %or:_(s1) = G_OR %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s64) = G_ZEXT %or(s1) + ; CHECK-NEXT: $x0 = COPY %zext(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(s64) = COPY $x0 + %y:_(s64) = COPY $x1 + %zero:_(s64) = G_CONSTANT i64 0 + %one:_(s64) = G_CONSTANT i64 1 + %cmp1:_(s1) = G_ICMP intpred(ne), %x:_(s64), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(ne), %y:_(s64), %one:_ + %or:_(s1) = G_OR %cmp1, %cmp2 + %zext:_(s64) = G_ZEXT %or:_(s1) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_or_eq_0_ne_0_s64 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_or_eq_0_ne_0_s64 + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s64) = COPY $x0 + ; CHECK-NEXT: %y:_(s64) = COPY $x1 + ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(eq), %x(s64), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(ne), %y(s64), %zero + ; CHECK-NEXT: %or:_(s1) = G_OR %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s64) = G_ZEXT %or(s1) + ; CHECK-NEXT: $x0 = COPY %zext(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(s64) = COPY $x0 + %y:_(s64) = COPY $x1 + %zero:_(s64) = G_CONSTANT i64 0 + %cmp1:_(s1) = G_ICMP intpred(eq), %x:_(s64), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(ne), %y:_(s64), %zero:_ + %or:_(s1) = G_OR %cmp1, %cmp2 + %zext:_(s64) = G_ZEXT %or:_(s1) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_or_ne_0_eq_0_s64 +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_or_ne_0_eq_0_s64 + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s64) = COPY $x0 + ; CHECK-NEXT: %y:_(s64) = COPY $x1 + ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(ne), %x(s64), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(eq), %y(s64), %zero + ; CHECK-NEXT: %or:_(s1) = G_OR %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s64) = G_ZEXT %or(s1) + ; CHECK-NEXT: $x0 = COPY %zext(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(s64) = COPY $x0 + %y:_(s64) = COPY $x1 + %zero:_(s64) = G_CONSTANT i64 0 + %cmp1:_(s1) = G_ICMP intpred(ne), %x:_(s64), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(eq), %y:_(s64), %zero:_ + %or:_(s1) = G_OR %cmp1, %cmp2 + %zext:_(s64) = G_ZEXT %or:_(s1) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: valid_and_eq_0_eq_0_vec +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: valid_and_eq_0_eq_0_vec + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(<2 x s32>) = COPY $x0 + ; CHECK-NEXT: %y:_(<2 x s32>) = COPY $x1 + ; CHECK-NEXT: %zero_scalar:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<2 x s32>) = G_OR %x, %y + ; CHECK-NEXT: %and:_(<2 x s1>) = G_ICMP intpred(eq), [[OR]](<2 x s32>), %zero + ; CHECK-NEXT: %zext:_(<2 x s32>) = G_ZEXT %and(<2 x s1>) + ; CHECK-NEXT: $x0 = COPY %zext(<2 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(<2 x s32>) = COPY $x0 + %y:_(<2 x s32>) = COPY $x1 + %zero_scalar:_(s32) = G_CONSTANT i32 0 + %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x:_(<2 x s32>), %zero:_ + %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y:_(<2 x s32>), %zero:_ + %and:_(<2 x s1>) = G_AND %cmp1, %cmp2 + %zext:_(<2 x s32>) = G_ZEXT %and:_(<2 x s1>) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_and_eq_non_0_eq_0_vec +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_and_eq_non_0_eq_0_vec + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(<2 x s32>) = COPY $x0 + ; CHECK-NEXT: %y:_(<2 x s32>) = COPY $x1 + ; CHECK-NEXT: %scalar0:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %scalar1:_(s32) = G_CONSTANT i32 1 + ; CHECK-NEXT: %zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar0(s32) + ; CHECK-NEXT: %non_zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar1(s32) + ; CHECK-NEXT: %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x(<2 x s32>), %non_zero + ; CHECK-NEXT: %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y(<2 x s32>), %zero + ; CHECK-NEXT: %and:_(<2 x s1>) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(<2 x s32>) = G_ZEXT %and(<2 x s1>) + ; CHECK-NEXT: $x0 = COPY %zext(<2 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(<2 x s32>) = COPY $x0 + %y:_(<2 x s32>) = COPY $x1 + %scalar0:_(s32) = G_CONSTANT i32 0 + %scalar1:_(s32) = G_CONSTANT i32 1 + %zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar0(s32) + %non_zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar1(s32) + %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x:_(<2 x s32>), %non_zero:_ + %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y:_(<2 x s32>), %zero:_ + %and:_(<2 x s1>) = G_AND %cmp1, %cmp2 + %zext:_(<2 x s32>) = G_ZEXT %and:_(<2 x s1>) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_and_eq_0_eq_non_0_vec +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_and_eq_0_eq_non_0_vec + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(<2 x s32>) = COPY $x0 + ; CHECK-NEXT: %y:_(<2 x s32>) = COPY $x1 + ; CHECK-NEXT: %scalar0:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %scalar1:_(s32) = G_CONSTANT i32 1 + ; CHECK-NEXT: %zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar0(s32) + ; CHECK-NEXT: %non_zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar1(s32) + ; CHECK-NEXT: %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x(<2 x s32>), %zero + ; CHECK-NEXT: %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y(<2 x s32>), %non_zero + ; CHECK-NEXT: %and:_(<2 x s1>) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(<2 x s32>) = G_ZEXT %and(<2 x s1>) + ; CHECK-NEXT: $x0 = COPY %zext(<2 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(<2 x s32>) = COPY $x0 + %y:_(<2 x s32>) = COPY $x1 + %scalar0:_(s32) = G_CONSTANT i32 0 + %scalar1:_(s32) = G_CONSTANT i32 1 + %zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar0(s32) + %non_zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar1(s32) + %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x:_(<2 x s32>), %zero:_ + %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y:_(<2 x s32>), %non_zero:_ + %and:_(<2 x s1>) = G_AND %cmp1, %cmp2 + %zext:_(<2 x s32>) = G_ZEXT %and:_(<2 x s1>) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_and_ne_0_eq_0_vec +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_and_ne_0_eq_0_vec + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(<2 x s32>) = COPY $x0 + ; CHECK-NEXT: %y:_(<2 x s32>) = COPY $x1 + ; CHECK-NEXT: %zero_scalar:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + ; CHECK-NEXT: %cmp1:_(<2 x s1>) = G_ICMP intpred(ne), %x(<2 x s32>), %zero + ; CHECK-NEXT: %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y(<2 x s32>), %zero + ; CHECK-NEXT: %and:_(<2 x s1>) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(<2 x s32>) = G_ZEXT %and(<2 x s1>) + ; CHECK-NEXT: $x0 = COPY %zext(<2 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(<2 x s32>) = COPY $x0 + %y:_(<2 x s32>) = COPY $x1 + %zero_scalar:_(s32) = G_CONSTANT i32 0 + %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + %cmp1:_(<2 x s1>) = G_ICMP intpred(ne), %x:_(<2 x s32>), %zero:_ + %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y:_(<2 x s32>), %zero:_ + %and:_(<2 x s1>) = G_AND %cmp1, %cmp2 + %zext:_(<2 x s32>) = G_ZEXT %and:_(<2 x s1>) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_and_eq_0_ne_0_vec +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_and_eq_0_ne_0_vec + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(<2 x s32>) = COPY $x0 + ; CHECK-NEXT: %y:_(<2 x s32>) = COPY $x1 + ; CHECK-NEXT: %zero_scalar:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + ; CHECK-NEXT: %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x(<2 x s32>), %zero + ; CHECK-NEXT: %cmp2:_(<2 x s1>) = G_ICMP intpred(ne), %y(<2 x s32>), %zero + ; CHECK-NEXT: %and:_(<2 x s1>) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(<2 x s32>) = G_ZEXT %and(<2 x s1>) + ; CHECK-NEXT: $x0 = COPY %zext(<2 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(<2 x s32>) = COPY $x0 + %y:_(<2 x s32>) = COPY $x1 + %zero_scalar:_(s32) = G_CONSTANT i32 0 + %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x:_(<2 x s32>), %zero:_ + %cmp2:_(<2 x s1>) = G_ICMP intpred(ne), %y:_(<2 x s32>), %zero:_ + %and:_(<2 x s1>) = G_AND %cmp1, %cmp2 + %zext:_(<2 x s32>) = G_ZEXT %and:_(<2 x s1>) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_and_ne_0_ne_0_vec +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_and_ne_0_ne_0_vec + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(<2 x s32>) = COPY $x0 + ; CHECK-NEXT: %y:_(<2 x s32>) = COPY $x1 + ; CHECK-NEXT: %zero_scalar:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + ; CHECK-NEXT: %cmp1:_(<2 x s1>) = G_ICMP intpred(ne), %x(<2 x s32>), %zero + ; CHECK-NEXT: %cmp2:_(<2 x s1>) = G_ICMP intpred(ne), %y(<2 x s32>), %zero + ; CHECK-NEXT: %and:_(<2 x s1>) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(<2 x s32>) = G_ZEXT %and(<2 x s1>) + ; CHECK-NEXT: $x0 = COPY %zext(<2 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(<2 x s32>) = COPY $x0 + %y:_(<2 x s32>) = COPY $x1 + %zero_scalar:_(s32) = G_CONSTANT i32 0 + %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + %cmp1:_(<2 x s1>) = G_ICMP intpred(ne), %x:_(<2 x s32>), %zero:_ + %cmp2:_(<2 x s1>) = G_ICMP intpred(ne), %y:_(<2 x s32>), %zero:_ + %and:_(<2 x s1>) = G_AND %cmp1, %cmp2 + %zext:_(<2 x s32>) = G_ZEXT %and:_(<2 x s1>) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: valid_or_ne_0_ne_0_vec +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: valid_or_ne_0_ne_0_vec + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(<2 x s32>) = COPY $x0 + ; CHECK-NEXT: %y:_(<2 x s32>) = COPY $x1 + ; CHECK-NEXT: %zero_scalar:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<2 x s32>) = G_OR %x, %y + ; CHECK-NEXT: %and:_(<2 x s1>) = G_ICMP intpred(ne), [[OR]](<2 x s32>), %zero + ; CHECK-NEXT: %zext:_(<2 x s32>) = G_ZEXT %and(<2 x s1>) + ; CHECK-NEXT: $x0 = COPY %zext(<2 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(<2 x s32>) = COPY $x0 + %y:_(<2 x s32>) = COPY $x1 + %zero_scalar:_(s32) = G_CONSTANT i32 0 + %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + %cmp1:_(<2 x s1>) = G_ICMP intpred(ne), %x:_(<2 x s32>), %zero:_ + %cmp2:_(<2 x s1>) = G_ICMP intpred(ne), %y:_(<2 x s32>), %zero:_ + %and:_(<2 x s1>) = G_OR %cmp1, %cmp2 + %zext:_(<2 x s32>) = G_ZEXT %and:_(<2 x s1>) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_or_ne_non_0_ne_0_vec +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_or_ne_non_0_ne_0_vec + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(<2 x s32>) = COPY $x0 + ; CHECK-NEXT: %y:_(<2 x s32>) = COPY $x1 + ; CHECK-NEXT: %scalar0:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %scalar1:_(s32) = G_CONSTANT i32 1 + ; CHECK-NEXT: %zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar0(s32) + ; CHECK-NEXT: %non_zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar1(s32) + ; CHECK-NEXT: %cmp1:_(<2 x s1>) = G_ICMP intpred(ne), %x(<2 x s32>), %non_zero + ; CHECK-NEXT: %cmp2:_(<2 x s1>) = G_ICMP intpred(ne), %y(<2 x s32>), %zero + ; CHECK-NEXT: %and:_(<2 x s1>) = G_OR %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(<2 x s32>) = G_ZEXT %and(<2 x s1>) + ; CHECK-NEXT: $x0 = COPY %zext(<2 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(<2 x s32>) = COPY $x0 + %y:_(<2 x s32>) = COPY $x1 + %scalar0:_(s32) = G_CONSTANT i32 0 + %scalar1:_(s32) = G_CONSTANT i32 1 + %zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar0(s32) + %non_zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar1(s32) + %cmp1:_(<2 x s1>) = G_ICMP intpred(ne), %x:_(<2 x s32>), %non_zero:_ + %cmp2:_(<2 x s1>) = G_ICMP intpred(ne), %y:_(<2 x s32>), %zero:_ + %and:_(<2 x s1>) = G_OR %cmp1, %cmp2 + %zext:_(<2 x s32>) = G_ZEXT %and:_(<2 x s1>) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_or_ne_0_ne_non_0_vec +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_or_ne_0_ne_non_0_vec + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(<2 x s32>) = COPY $x0 + ; CHECK-NEXT: %y:_(<2 x s32>) = COPY $x1 + ; CHECK-NEXT: %scalar0:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %scalar1:_(s32) = G_CONSTANT i32 1 + ; CHECK-NEXT: %zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar0(s32) + ; CHECK-NEXT: %non_zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar1(s32) + ; CHECK-NEXT: %cmp1:_(<2 x s1>) = G_ICMP intpred(ne), %x(<2 x s32>), %zero + ; CHECK-NEXT: %cmp2:_(<2 x s1>) = G_ICMP intpred(ne), %y(<2 x s32>), %non_zero + ; CHECK-NEXT: %and:_(<2 x s1>) = G_OR %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(<2 x s32>) = G_ZEXT %and(<2 x s1>) + ; CHECK-NEXT: $x0 = COPY %zext(<2 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(<2 x s32>) = COPY $x0 + %y:_(<2 x s32>) = COPY $x1 + %scalar0:_(s32) = G_CONSTANT i32 0 + %scalar1:_(s32) = G_CONSTANT i32 1 + %zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar0(s32) + %non_zero:_(<2 x s32>) = G_BUILD_VECTOR %scalar0(s32), %scalar1(s32) + %cmp1:_(<2 x s1>) = G_ICMP intpred(ne), %x:_(<2 x s32>), %zero:_ + %cmp2:_(<2 x s1>) = G_ICMP intpred(ne), %y:_(<2 x s32>), %non_zero:_ + %and:_(<2 x s1>) = G_OR %cmp1, %cmp2 + %zext:_(<2 x s32>) = G_ZEXT %and:_(<2 x s1>) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_or_eq_0_ne_0_vec +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_or_eq_0_ne_0_vec + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(<2 x s32>) = COPY $x0 + ; CHECK-NEXT: %y:_(<2 x s32>) = COPY $x1 + ; CHECK-NEXT: %zero_scalar:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + ; CHECK-NEXT: %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x(<2 x s32>), %zero + ; CHECK-NEXT: %cmp2:_(<2 x s1>) = G_ICMP intpred(ne), %y(<2 x s32>), %zero + ; CHECK-NEXT: %and:_(<2 x s1>) = G_OR %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(<2 x s32>) = G_ZEXT %and(<2 x s1>) + ; CHECK-NEXT: $x0 = COPY %zext(<2 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(<2 x s32>) = COPY $x0 + %y:_(<2 x s32>) = COPY $x1 + %zero_scalar:_(s32) = G_CONSTANT i32 0 + %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x:_(<2 x s32>), %zero:_ + %cmp2:_(<2 x s1>) = G_ICMP intpred(ne), %y:_(<2 x s32>), %zero:_ + %and:_(<2 x s1>) = G_OR %cmp1, %cmp2 + %zext:_(<2 x s32>) = G_ZEXT %and:_(<2 x s1>) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_or_ne_0_eq_0_vec +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_or_ne_0_eq_0_vec + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(<2 x s32>) = COPY $x0 + ; CHECK-NEXT: %y:_(<2 x s32>) = COPY $x1 + ; CHECK-NEXT: %zero_scalar:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + ; CHECK-NEXT: %cmp1:_(<2 x s1>) = G_ICMP intpred(ne), %x(<2 x s32>), %zero + ; CHECK-NEXT: %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y(<2 x s32>), %zero + ; CHECK-NEXT: %and:_(<2 x s1>) = G_OR %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(<2 x s32>) = G_ZEXT %and(<2 x s1>) + ; CHECK-NEXT: $x0 = COPY %zext(<2 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(<2 x s32>) = COPY $x0 + %y:_(<2 x s32>) = COPY $x1 + %zero_scalar:_(s32) = G_CONSTANT i32 0 + %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + %cmp1:_(<2 x s1>) = G_ICMP intpred(ne), %x:_(<2 x s32>), %zero:_ + %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y:_(<2 x s32>), %zero:_ + %and:_(<2 x s1>) = G_OR %cmp1, %cmp2 + %zext:_(<2 x s32>) = G_ZEXT %and:_(<2 x s1>) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_or_eq_0_eq_0_vec +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_or_eq_0_eq_0_vec + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(<2 x s32>) = COPY $x0 + ; CHECK-NEXT: %y:_(<2 x s32>) = COPY $x1 + ; CHECK-NEXT: %zero_scalar:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + ; CHECK-NEXT: %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x(<2 x s32>), %zero + ; CHECK-NEXT: %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y(<2 x s32>), %zero + ; CHECK-NEXT: %and:_(<2 x s1>) = G_OR %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(<2 x s32>) = G_ZEXT %and(<2 x s1>) + ; CHECK-NEXT: $x0 = COPY %zext(<2 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(<2 x s32>) = COPY $x0 + %y:_(<2 x s32>) = COPY $x1 + %zero_scalar:_(s32) = G_CONSTANT i32 0 + %zero:_(<2 x s32>) = G_BUILD_VECTOR %zero_scalar(s32), %zero_scalar(s32) + %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x:_(<2 x s32>), %zero:_ + %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y:_(<2 x s32>), %zero:_ + %and:_(<2 x s1>) = G_OR %cmp1, %cmp2 + %zext:_(<2 x s32>) = G_ZEXT %and:_(<2 x s1>) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_p0_src +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $x1 + + ; CHECK-LABEL: name: invalid_p0_src + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(p0) = COPY $x0 + ; CHECK-NEXT: %y:_(p0) = COPY $x1 + ; CHECK-NEXT: %zero:_(p0) = G_CONSTANT i64 0 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(eq), %x(p0), %zero + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(eq), %y(p0), %zero + ; CHECK-NEXT: %and:_(s1) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s64) = G_ZEXT %and(s1) + ; CHECK-NEXT: $x0 = COPY %zext(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(p0) = COPY $x0 + %y:_(p0) = COPY $x1 + %zero:_(p0) = G_CONSTANT i64 0 + %cmp1:_(s1) = G_ICMP intpred(eq), %x:_(p0), %zero:_ + %cmp2:_(s1) = G_ICMP intpred(eq), %y:_(p0), %zero:_ + %and:_(s1) = G_AND %cmp1, %cmp2 + %zext:_(s64) = G_ZEXT %and:_(s1) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_p0_src_vec +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $q0, $q1 + + ; CHECK-LABEL: name: invalid_p0_src_vec + ; CHECK: liveins: $q0, $q1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(<2 x p0>) = COPY $q0 + ; CHECK-NEXT: %y:_(<2 x p0>) = COPY $q1 + ; CHECK-NEXT: %scalar0:_(p0) = G_CONSTANT i64 0 + ; CHECK-NEXT: %zero:_(<2 x p0>) = G_BUILD_VECTOR %scalar0(p0), %scalar0(p0) + ; CHECK-NEXT: %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x(<2 x p0>), %zero + ; CHECK-NEXT: %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y(<2 x p0>), %zero + ; CHECK-NEXT: %or:_(<2 x s1>) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(<2 x s64>) = G_ZEXT %or(<2 x s1>) + ; CHECK-NEXT: $q0 = COPY %zext(<2 x s64>) + ; CHECK-NEXT: RET_ReallyLR implicit $q0 + %x:_(<2 x p0>) = COPY $q0 + %y:_(<2 x p0>) = COPY $q1 + %scalar0:_(p0) = G_CONSTANT i64 0 + %zero:_(<2 x p0>) = G_BUILD_VECTOR %scalar0(p0), %scalar0(p0) + %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x:_(<2 x p0>), %zero:_ + %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y:_(<2 x p0>), %zero:_ + %or:_(<2 x s1>) = G_AND %cmp1, %cmp2 + %zext:_(<2 x s64>) = G_ZEXT %or:_(<2 x s1>) + $q0 = COPY %zext + RET_ReallyLR implicit $q0 + +... +--- +name: invalid_diff_src_ty +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $w0, $x1 + + ; CHECK-LABEL: name: invalid_diff_src_ty + ; CHECK: liveins: $w0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s32) = COPY $w0 + ; CHECK-NEXT: %y:_(s64) = COPY $x1 + ; CHECK-NEXT: %zero_s32:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %zero_s64:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: %cmp1:_(s1) = G_ICMP intpred(eq), %x(s32), %zero_s32 + ; CHECK-NEXT: %cmp2:_(s1) = G_ICMP intpred(eq), %y(s64), %zero_s64 + ; CHECK-NEXT: %and:_(s1) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(s64) = G_ZEXT %and(s1) + ; CHECK-NEXT: $x0 = COPY %zext(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(s32) = COPY $w0 + %y:_(s64) = COPY $x1 + %zero_s32:_(s32) = G_CONSTANT i32 0 + %zero_s64:_(s64) = G_CONSTANT i64 0 + %cmp1:_(s1) = G_ICMP intpred(eq), %x:_(s32), %zero_s32:_ + %cmp2:_(s1) = G_ICMP intpred(eq), %y:_(s64), %zero_s64:_ + %and:_(s1) = G_AND %cmp1, %cmp2 + %zext:_(s64) = G_ZEXT %and:_(s1) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... +--- +name: invalid_diff_src_ty_vec +tracksRegLiveness: true +legalized: true +body: | + bb.0: + liveins: $x0, $q1 + + ; CHECK-LABEL: name: invalid_diff_src_ty_vec + ; CHECK: liveins: $x0, $q1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(<2 x s32>) = COPY $x0 + ; CHECK-NEXT: %y:_(<2 x s64>) = COPY $q1 + ; CHECK-NEXT: %scalar0s32:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: %scalar0s64:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: %zero_s32:_(<2 x s32>) = G_BUILD_VECTOR %scalar0s32(s32), %scalar0s32(s32) + ; CHECK-NEXT: %zero_s64:_(<2 x s64>) = G_BUILD_VECTOR %scalar0s64(s64), %scalar0s64(s64) + ; CHECK-NEXT: %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x(<2 x s32>), %zero_s32 + ; CHECK-NEXT: %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y(<2 x s64>), %zero_s64 + ; CHECK-NEXT: %or:_(<2 x s1>) = G_AND %cmp1, %cmp2 + ; CHECK-NEXT: %zext:_(<2 x s32>) = G_ZEXT %or(<2 x s1>) + ; CHECK-NEXT: $x0 = COPY %zext(<2 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(<2 x s32>) = COPY $x0 + %y:_(<2 x s64>) = COPY $q1 + %scalar0s32:_(s32) = G_CONSTANT i32 0 + %scalar0s64:_(s64) = G_CONSTANT i64 0 + %zero_s32:_(<2 x s32>) = G_BUILD_VECTOR %scalar0s32(s32), %scalar0s32(s32) + %zero_s64:_(<2 x s64>) = G_BUILD_VECTOR %scalar0s64(s64), %scalar0s64(s64) + %cmp1:_(<2 x s1>) = G_ICMP intpred(eq), %x:_(<2 x s32>), %zero_s32:_ + %cmp2:_(<2 x s1>) = G_ICMP intpred(eq), %y:_(<2 x s64>), %zero_s64:_ + %or:_(<2 x s1>) = G_AND %cmp1, %cmp2 + %zext:_(<2 x s32>) = G_ZEXT %or:_(<2 x s1>) + $x0 = COPY %zext + RET_ReallyLR implicit $x0 + +... diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll index ef8e466..42f6570 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll @@ -26,7 +26,7 @@ define void @asm_simple_register_clobber() { define i64 @asm_register_early_clobber() { ; CHECK-LABEL: name: asm_register_early_clobber ; CHECK: bb.1 (%ir-block.0): - ; CHECK-NEXT: INLINEASM &"mov $0, 7; mov $1, 7", 1 /* sideeffect attdialect */, 2752523 /* regdef-ec:GPR64common */, def early-clobber %0, 2752523 /* regdef-ec:GPR64common */, def early-clobber %1, !0 + ; CHECK-NEXT: INLINEASM &"mov $0, 7; mov $1, 7", 1 /* sideeffect attdialect */, {{[0-9]+}} /* regdef-ec:GPR64common */, def early-clobber %0, {{[0-9]+}} /* regdef-ec:GPR64common */, def early-clobber %1, !0 ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY %0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY %1 ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY]], [[COPY1]] @@ -54,7 +54,7 @@ entry: define i32 @test_single_register_output() nounwind ssp { ; CHECK-LABEL: name: test_single_register_output ; CHECK: bb.1.entry: - ; CHECK-NEXT: INLINEASM &"mov ${0:w}, 7", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %0 + ; CHECK-NEXT: INLINEASM &"mov ${0:w}, 7", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %0 ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY %0 ; CHECK-NEXT: $w0 = COPY [[COPY]](s32) ; CHECK-NEXT: RET_ReallyLR implicit $w0 @@ -66,7 +66,7 @@ entry: define i64 @test_single_register_output_s64() nounwind ssp { ; CHECK-LABEL: name: test_single_register_output_s64 ; CHECK: bb.1.entry: - ; CHECK-NEXT: INLINEASM &"mov $0, 7", 0 /* attdialect */, 2752522 /* regdef:GPR64common */, def %0 + ; CHECK-NEXT: INLINEASM &"mov $0, 7", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR64common */, def %0 ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY %0 ; CHECK-NEXT: $x0 = COPY [[COPY]](s64) ; CHECK-NEXT: RET_ReallyLR implicit $x0 @@ -79,7 +79,7 @@ entry: define float @test_multiple_register_outputs_same() #0 { ; CHECK-LABEL: name: test_multiple_register_outputs_same ; CHECK: bb.1 (%ir-block.0): - ; CHECK-NEXT: INLINEASM &"mov $0, #0; mov $1, #0", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %0, 1703946 /* regdef:GPR32common */, def %1 + ; CHECK-NEXT: INLINEASM &"mov $0, #0; mov $1, #0", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %0, {{[0-9]+}} /* regdef:GPR32common */, def %1 ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY %0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY %1 ; CHECK-NEXT: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[COPY]], [[COPY1]] @@ -96,7 +96,7 @@ define float @test_multiple_register_outputs_same() #0 { define double @test_multiple_register_outputs_mixed() #0 { ; CHECK-LABEL: name: test_multiple_register_outputs_mixed ; CHECK: bb.1 (%ir-block.0): - ; CHECK-NEXT: INLINEASM &"mov $0, #0; mov $1, #0", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %0, 2555914 /* regdef:FPR64 */, def %1 + ; CHECK-NEXT: INLINEASM &"mov $0, #0; mov $1, #0", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %0, {{[0-9]+}} /* regdef:FPR64 */, def %1 ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY %0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY %1 ; CHECK-NEXT: $d0 = COPY [[COPY1]](s64) @@ -125,7 +125,7 @@ define zeroext i8 @test_register_output_trunc(ptr %src) nounwind { ; CHECK-NEXT: liveins: $x0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 - ; CHECK-NEXT: INLINEASM &"mov ${0:w}, 32", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %1 + ; CHECK-NEXT: INLINEASM &"mov ${0:w}, 32", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %1 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY %1 ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32) ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s8) @@ -155,7 +155,7 @@ define void @test_input_register_imm() { ; CHECK: bb.1 (%ir-block.0): ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 42 ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY [[C]](s64) - ; CHECK-NEXT: INLINEASM &"mov x0, $0", 1 /* sideeffect attdialect */, 2752521 /* reguse:GPR64common */, [[COPY]] + ; CHECK-NEXT: INLINEASM &"mov x0, $0", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:GPR64common */, [[COPY]] ; CHECK-NEXT: RET_ReallyLR call void asm sideeffect "mov x0, $0", "r"(i64 42) ret void @@ -190,7 +190,7 @@ define zeroext i8 @test_input_register(ptr %src) nounwind { ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY [[COPY]](p0) - ; CHECK-NEXT: INLINEASM &"ldtrb ${0:w}, [$1]", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %1, 2752521 /* reguse:GPR64common */, [[COPY1]] + ; CHECK-NEXT: INLINEASM &"ldtrb ${0:w}, [$1]", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %1, {{[0-9]+}} /* reguse:GPR64common */, [[COPY1]] ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY %1 ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY2]](s32) ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s8) @@ -207,7 +207,7 @@ define i32 @test_memory_constraint(ptr %a) nounwind { ; CHECK-NEXT: liveins: $x0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 - ; CHECK-NEXT: INLINEASM &"ldr $0, $1", 8 /* mayload attdialect */, 1703946 /* regdef:GPR32common */, def %1, 262158 /* mem:m */, [[COPY]](p0) + ; CHECK-NEXT: INLINEASM &"ldr $0, $1", 8 /* mayload attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %1, 262158 /* mem:m */, [[COPY]](p0) ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY %1 ; CHECK-NEXT: $w0 = COPY [[COPY1]](s32) ; CHECK-NEXT: RET_ReallyLR implicit $w0 @@ -221,7 +221,7 @@ define i16 @test_anyext_input() { ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 1 ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s16) ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY [[ANYEXT]](s32) - ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, 1703946 /* regdef:GPR32common */, def %0, 1703945 /* reguse:GPR32common */, [[COPY]] + ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %0, {{[0-9]+}} /* reguse:GPR32common */, [[COPY]] ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY %0 ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32) ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16) @@ -237,7 +237,7 @@ define i16 @test_anyext_input_with_matching_constraint() { ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 1 ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s16) ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY [[ANYEXT]](s32) - ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, 1703946 /* regdef:GPR32common */, def %0, 2147483657 /* reguse tiedto:$0 */, [[COPY]](tied-def 3) + ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %0, 2147483657 /* reguse tiedto:$0 */, [[COPY]](tied-def 3) ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY %0 ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32) ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16) diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unwind-inline-asm.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unwind-inline-asm.ll index 59eb80a..fbffb50 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unwind-inline-asm.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unwind-inline-asm.ll @@ -71,7 +71,7 @@ define void @test2() #0 personality ptr @__gcc_personality_v0 { ; CHECK-NEXT: G_INVOKE_REGION_START ; CHECK-NEXT: EH_LABEL <mcsymbol > ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY [[DEF]](p0) - ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, 2752521 /* reguse:GPR64common */, [[COPY]] + ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:GPR64common */, [[COPY]] ; CHECK-NEXT: EH_LABEL <mcsymbol > ; CHECK-NEXT: G_BR %bb.2 ; CHECK-NEXT: {{ $}} diff --git a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll index 8ed7059..5829969 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll @@ -5,7 +5,7 @@ entry: ; CHECK: %0:ppr = COPY $p0 ; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2) ; CHECK: %1:pnr_p8to15 = COPY %0 -; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, 458761 /* reguse:PNR_p8to15 */, %1 +; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR_p8to15 */, %1 ; CHECK: RET_ReallyLR %predcnt.addr = alloca target("aarch64.svcount"), align 2 store target("aarch64.svcount") %predcnt, ptr %predcnt.addr, align 2 @@ -19,7 +19,7 @@ entry: ; CHECK: %0:ppr = COPY $p0 ; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2) ; CHECK: %1:pnr = COPY %0 -; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, 262153 /* reguse:PNR */, %1 +; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR */, %1 ; CHECK: RET_ReallyLR %predcnt.addr = alloca target("aarch64.svcount"), align 2 store target("aarch64.svcount") %predcnt, ptr %predcnt.addr, align 2 @@ -33,7 +33,7 @@ entry: ; CHECK: %0:ppr = COPY $p0 ; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2) ; CHECK: %1:pnr_3b = COPY %0 -; CHECK: INLINEASM &"fadd z0.h, $0/m, z0.h, #0.5", 1 /* sideeffect attdialect */, 393225 /* reguse:PNR_3b */, %1 +; CHECK: INLINEASM &"fadd z0.h, $0/m, z0.h, #0.5", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR_3b */, %1 ; CHECK: RET_ReallyLR %predcnt.addr = alloca target("aarch64.svcount"), align 2 store target("aarch64.svcount") %predcnt, ptr %predcnt.addr, align 2 diff --git a/llvm/test/CodeGen/AArch64/aes.ll b/llvm/test/CodeGen/AArch64/aes.ll new file mode 100644 index 0000000..2bef28d --- /dev/null +++ b/llvm/test/CodeGen/AArch64/aes.ll @@ -0,0 +1,43 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc %s -o - -mtriple=aarch64 -mattr=+aes | FileCheck %s + +declare <16 x i8> @llvm.aarch64.crypto.aese(<16 x i8> %d, <16 x i8> %k) +declare <16 x i8> @llvm.aarch64.crypto.aesd(<16 x i8> %d, <16 x i8> %k) + +define <16 x i8> @aese(<16 x i8> %a, <16 x i8> %b) { +; CHECK-LABEL: aese: +; CHECK: // %bb.0: +; CHECK-NEXT: aese v0.16b, v1.16b +; CHECK-NEXT: ret + %r = call <16 x i8> @llvm.aarch64.crypto.aese(<16 x i8> %a, <16 x i8> %b) + ret <16 x i8> %r +} + +define <16 x i8> @aese_c(<16 x i8> %a, <16 x i8> %b) { +; CHECK-LABEL: aese_c: +; CHECK: // %bb.0: +; CHECK-NEXT: aese v1.16b, v0.16b +; CHECK-NEXT: mov v0.16b, v1.16b +; CHECK-NEXT: ret + %r = call <16 x i8> @llvm.aarch64.crypto.aese(<16 x i8> %b, <16 x i8> %a) + ret <16 x i8> %r +} + +define <16 x i8> @aesd(<16 x i8> %a, <16 x i8> %b) { +; CHECK-LABEL: aesd: +; CHECK: // %bb.0: +; CHECK-NEXT: aesd v0.16b, v1.16b +; CHECK-NEXT: ret + %r = call <16 x i8> @llvm.aarch64.crypto.aesd(<16 x i8> %a, <16 x i8> %b) + ret <16 x i8> %r +} + +define <16 x i8> @aesd_c(<16 x i8> %a, <16 x i8> %b) { +; CHECK-LABEL: aesd_c: +; CHECK: // %bb.0: +; CHECK-NEXT: aesd v1.16b, v0.16b +; CHECK-NEXT: mov v0.16b, v1.16b +; CHECK-NEXT: ret + %r = call <16 x i8> @llvm.aarch64.crypto.aesd(<16 x i8> %b, <16 x i8> %a) + ret <16 x i8> %r +} diff --git a/llvm/test/CodeGen/AArch64/callbr-asm-outputs-indirect-isel.ll b/llvm/test/CodeGen/AArch64/callbr-asm-outputs-indirect-isel.ll index 3b7b5dd3..fbe89e7 100644 --- a/llvm/test/CodeGen/AArch64/callbr-asm-outputs-indirect-isel.ll +++ b/llvm/test/CodeGen/AArch64/callbr-asm-outputs-indirect-isel.ll @@ -18,7 +18,7 @@ define i32 @test0() { ; CHECK: bb.0.entry: ; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.1(0x00000000) ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: INLINEASM_BR &"# $0", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %5, 13 /* imm */, %bb.1 + ; CHECK-NEXT: INLINEASM_BR &"# $0", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %5, 13 /* imm */, %bb.1 ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32all = COPY %5 ; CHECK-NEXT: B %bb.2 ; CHECK-NEXT: {{ $}} @@ -31,7 +31,7 @@ define i32 @test0() { ; CHECK-NEXT: bb.2.direct: ; CHECK-NEXT: successors: %bb.4(0x80000000), %bb.3(0x00000000) ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: INLINEASM_BR &"# $0", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %7, 13 /* imm */, %bb.3 + ; CHECK-NEXT: INLINEASM_BR &"# $0", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %7, 13 /* imm */, %bb.3 ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32all = COPY %7 ; CHECK-NEXT: B %bb.4 ; CHECK-NEXT: {{ $}} @@ -107,7 +107,7 @@ define i32 @dont_split1() { ; CHECK: bb.0.entry: ; CHECK-NEXT: successors: %bb.1(0x80000000), %bb.2(0x00000000) ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %1, 13 /* imm */, %bb.2 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %1, 13 /* imm */, %bb.2 ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32all = COPY %1 ; CHECK-NEXT: B %bb.1 ; CHECK-NEXT: {{ $}} @@ -168,7 +168,7 @@ define i32 @dont_split3() { ; CHECK: bb.0.entry: ; CHECK-NEXT: successors: %bb.1(0x80000000), %bb.2(0x00000000) ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %0, 13 /* imm */, %bb.2 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %0, 13 /* imm */, %bb.2 ; CHECK-NEXT: B %bb.1 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.1.x: @@ -194,7 +194,7 @@ define i32 @split_me0() { ; CHECK: bb.0.entry: ; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.1(0x00000000) ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %3, 13 /* imm */, %bb.1 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %3, 13 /* imm */, %bb.1 ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32all = COPY %3 ; CHECK-NEXT: B %bb.2 ; CHECK-NEXT: {{ $}} @@ -244,7 +244,7 @@ define i32 @split_me1(i1 %z) { ; CHECK-NEXT: bb.1.w: ; CHECK-NEXT: successors: %bb.3(0x80000000), %bb.2(0x00000000) ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %5, 13 /* imm */, %bb.2, 13 /* imm */, %bb.2 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %5, 13 /* imm */, %bb.2, 13 /* imm */, %bb.2 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32all = COPY %5 ; CHECK-NEXT: B %bb.3 ; CHECK-NEXT: {{ $}} @@ -297,7 +297,7 @@ define i32 @split_me2(i1 %z) { ; CHECK-NEXT: bb.1.w: ; CHECK-NEXT: successors: %bb.3(0x80000000), %bb.2(0x00000000) ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %6, 13 /* imm */, %bb.2, 13 /* imm */, %bb.2 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %6, 13 /* imm */, %bb.2, 13 /* imm */, %bb.2 ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32all = COPY %6 ; CHECK-NEXT: B %bb.3 ; CHECK-NEXT: {{ $}} @@ -340,7 +340,7 @@ define i32 @dont_split4() { ; CHECK: bb.0.entry: ; CHECK-NEXT: successors: %bb.1(0x80000000), %bb.2(0x00000000) ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %3, 13 /* imm */, %bb.2 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %3, 13 /* imm */, %bb.2 ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32all = COPY %3 ; CHECK-NEXT: B %bb.1 ; CHECK-NEXT: {{ $}} @@ -379,7 +379,7 @@ define i32 @dont_split5() { ; CHECK: bb.0.entry: ; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.1(0x00000000) ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %3, 13 /* imm */, %bb.1 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %3, 13 /* imm */, %bb.1 ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32all = COPY %3 ; CHECK-NEXT: B %bb.2 ; CHECK-NEXT: {{ $}} @@ -410,7 +410,7 @@ define i32 @split_me3() { ; CHECK: bb.0.entry: ; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.1(0x00000000) ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %3, 13 /* imm */, %bb.1 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %3, 13 /* imm */, %bb.1 ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32all = COPY %3 ; CHECK-NEXT: B %bb.2 ; CHECK-NEXT: {{ $}} @@ -456,7 +456,7 @@ define i32 @dont_split6(i32 %0) { ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32all = PHI [[COPY]], %bb.0, %2, %bb.2 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32common = COPY [[PHI]] - ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %4, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3), 13 /* imm */, %bb.2 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %4, 2147483657 /* reguse tiedto:$0 */, [[COPY1]](tied-def 3), 13 /* imm */, %bb.2 ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32all = COPY %4 ; CHECK-NEXT: B %bb.3 ; CHECK-NEXT: {{ $}} @@ -491,7 +491,7 @@ define i32 @split_me4() { ; CHECK: bb.0.entry: ; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.1(0x00000000) ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %3, 13 /* imm */, %bb.1 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %3, 13 /* imm */, %bb.1 ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32all = COPY %3 ; CHECK-NEXT: B %bb.2 ; CHECK-NEXT: {{ $}} @@ -522,7 +522,7 @@ define i32 @split_me5() { ; CHECK: bb.0.entry: ; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.1(0x00000000) ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 1703946 /* regdef:GPR32common */, def %3, 13 /* imm */, %bb.1 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, {{[0-9]+}} /* regdef:GPR32common */, def %3, 13 /* imm */, %bb.1 ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32all = COPY %3 ; CHECK-NEXT: B %bb.2 ; CHECK-NEXT: {{ $}} diff --git a/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir b/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir index 483dbd2..92fb053 100644 --- a/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir +++ b/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir @@ -91,10 +91,10 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[LOADgot:%[0-9]+]]:gpr64common = LOADgot target-flags(aarch64-got) @c ; CHECK-NEXT: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[LOADgot]], 0 :: (dereferenceable load (s64) from @c) - ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, 2359306 /* regdef:WSeqPairsClass_with_sube32_in_MatrixIndexGPR32_12_15 */, def %2, 2147483657 /* reguse tiedto:$0 */, [[LDRDui]](tied-def 3) + ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, {{[0-9]+}} /* regdef:WSeqPairsClass_with_sube32_in_MatrixIndexGPR32_12_15 */, def %2, 2147483657 /* reguse tiedto:$0 */, [[LDRDui]](tied-def 3) ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr64 = COPY %2 ; CHECK-NEXT: [[LDRDui1:%[0-9]+]]:fpr64 = LDRDui [[LOADgot]], 0 :: (dereferenceable load (s64) from @c) - ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, 2359306 /* regdef:WSeqPairsClass_with_sube32_in_MatrixIndexGPR32_12_15 */, def %4, 2147483657 /* reguse tiedto:$0 */, [[LDRDui1]](tied-def 3) + ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, {{[0-9]+}} /* regdef:WSeqPairsClass_with_sube32_in_MatrixIndexGPR32_12_15 */, def %4, 2147483657 /* reguse tiedto:$0 */, [[LDRDui1]](tied-def 3) ; CHECK-NEXT: [[FNEGDr:%[0-9]+]]:fpr64 = FNEGDr %2 ; CHECK-NEXT: nofpexcept FCMPDrr %4, killed [[FNEGDr]], implicit-def $nzcv, implicit $fpcr ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv diff --git a/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir b/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir index 041b2dc..6514834 100644 --- a/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir +++ b/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir @@ -487,7 +487,7 @@ body: | ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0 ; CHECK-NEXT: [[DEF:%[0-9]+]]:gpr64all = IMPLICIT_DEF ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY [[DEF]] - ; CHECK-NEXT: INLINEASM &"ldr ${0:s}, $1", 8 /* mayload attdialect */, 2359306 /* regdef:WSeqPairsClass_with_sube32_in_MatrixIndexGPR32_12_15 */, def %1, 262158 /* mem:m */, killed [[COPY1]] + ; CHECK-NEXT: INLINEASM &"ldr ${0:s}, $1", 8 /* mayload attdialect */, {{[0-9]+}} /* regdef:WSeqPairsClass_with_sube32_in_MatrixIndexGPR32_12_15 */, def %1, 262158 /* mem:m */, killed [[COPY1]] ; CHECK-NEXT: [[MOVIv2d_ns:%[0-9]+]]:fpr128 = MOVIv2d_ns 0 ; CHECK-NEXT: [[COPY2:%[0-9]+]]:fpr64 = COPY [[MOVIv2d_ns]].dsub ; CHECK-NEXT: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF diff --git a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll index 46b714d..bb9546a 100644 --- a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll +++ b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll @@ -21,9 +21,7 @@ define noundef i1 @logger(i32 noundef %logLevel, ptr %ea, ptr %pll) { ; CHECK-NEXT: ret ; CHECK-NEXT: .LBB1_2: // %land.rhs ; CHECK-NEXT: ldr x8, [x1] -; CHECK-NEXT: ldrb w8, [x8] -; CHECK-NEXT: cmp w8, #0 -; CHECK-NEXT: cset w0, ne +; CHECK-NEXT: ldrb w0, [x8] ; CHECK-NEXT: ret entry: %0 = load i32, ptr %pll, align 4 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll index 0f70c19..d4d5cb1 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc -global-isel -amdgpu-global-isel-risky-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s +; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s ; Divergent phis that don't require lowering using lane mask merging @@ -147,32 +147,28 @@ define void @divergent_i1_phi_used_inside_loop_bigger_loop_body(float %val, floa ; GFX10-LABEL: divergent_i1_phi_used_inside_loop_bigger_loop_body: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, 1.0, v1 -; GFX10-NEXT: s_mov_b32 s5, 0 +; GFX10-NEXT: s_mov_b32 s4, 0 +; GFX10-NEXT: v_cmp_lt_f32_e64 s5, 1.0, v1 ; GFX10-NEXT: v_mov_b32_e32 v1, 0x3e8 -; GFX10-NEXT: v_mov_b32_e32 v8, s5 +; GFX10-NEXT: v_mov_b32_e32 v8, s4 ; GFX10-NEXT: ; implicit-def: $sgpr6 -; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo ; GFX10-NEXT: s_branch .LBB3_2 ; GFX10-NEXT: .LBB3_1: ; %loop_body ; GFX10-NEXT: ; in Loop: Header=BB3_2 Depth=1 ; GFX10-NEXT: v_cvt_f32_u32_e32 v9, v8 -; GFX10-NEXT: s_xor_b32 s4, s4, -1 +; GFX10-NEXT: s_xor_b32 s5, s5, -1 ; GFX10-NEXT: v_add_nc_u32_e32 v8, 1, v8 ; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s4 -; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5 +; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4 ; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo -; GFX10-NEXT: s_and_b32 s4, exec_lo, s4 -; GFX10-NEXT: s_or_b32 s6, s6, s4 -; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5 +; GFX10-NEXT: s_and_b32 s7, exec_lo, s5 +; GFX10-NEXT: s_or_b32 s6, s6, s7 +; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4 ; GFX10-NEXT: s_cbranch_execz .LBB3_6 ; GFX10-NEXT: .LBB3_2: ; %loop_start ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX10-NEXT: v_and_b32_e32 v9, 1, v9 ; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, 0x3e8, v8 ; GFX10-NEXT: s_mov_b32 s7, 1 -; GFX10-NEXT: v_cmp_ne_u32_e64 s4, 0, v9 ; GFX10-NEXT: s_cbranch_vccz .LBB3_4 ; GFX10-NEXT: ; %bb.3: ; %else ; GFX10-NEXT: ; in Loop: Header=BB3_2 Depth=1 @@ -189,7 +185,7 @@ define void @divergent_i1_phi_used_inside_loop_bigger_loop_body(float %val, floa ; GFX10-NEXT: flat_store_dword v[4:5], v1 ; GFX10-NEXT: s_branch .LBB3_1 ; GFX10-NEXT: .LBB3_6: ; %exit -; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s6 ; GFX10-NEXT: flat_store_dword v[2:3], v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.mir index 5549c89..9b0bd27 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.mir @@ -33,6 +33,7 @@ body: | ; GFX10-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY2]](s32), [[C]] ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX10-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY3]](s32), [[C1]] + ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP]](s1) ; GFX10-NEXT: G_BRCOND [[ICMP1]](s1), %bb.2 ; GFX10-NEXT: G_BR %bb.1 ; GFX10-NEXT: {{ $}} @@ -46,7 +47,8 @@ body: | ; GFX10-NEXT: bb.2: ; GFX10-NEXT: successors: %bb.4(0x80000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = G_PHI %14(s1), %bb.3, [[ICMP]](s1), %bb.0 + ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY6]](s1), %bb.0, %20(s1), %bb.3 + ; GFX10-NEXT: [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1) ; GFX10-NEXT: G_BR %bb.4 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.3: @@ -54,12 +56,13 @@ body: | ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; GFX10-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY2]](s32), [[C3]] + ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1) ; GFX10-NEXT: G_BR %bb.2 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.4: ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2 ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 - ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[PHI]](s1), [[C5]], [[C4]] + ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY7]](s1), [[C5]], [[C4]] ; GFX10-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1) ; GFX10-NEXT: S_ENDPGM 0 bb.0: @@ -126,9 +129,10 @@ body: | ; GFX10-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr0 ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 6 ; GFX10-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY2]](s32), [[C]] - ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP]](s1) ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX10-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY3]](s32), [[C1]] + ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP]](s1) + ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[COPY4]](s1) ; GFX10-NEXT: G_BRCOND [[ICMP1]](s1), %bb.2 ; GFX10-NEXT: G_BR %bb.1 ; GFX10-NEXT: {{ $}} @@ -137,17 +141,17 @@ body: | ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; GFX10-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY2]](s32), [[C2]] - ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1) - ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY4]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY5]](s1), implicit-def $scc + ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1) + ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY5]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY6]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.2: - ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[ICMP]](s1), %bb.0, [[S_OR_B32_]](s1), %bb.1 - ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1) + ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY4]](s1), %bb.0, [[S_OR_B32_]](s1), %bb.1 + ; GFX10-NEXT: [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1) ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2 ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 - ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY6]](s1), [[C4]], [[C3]] + ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY7]](s1), [[C4]], [[C3]] ; GFX10-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1) ; GFX10-NEXT: S_ENDPGM 0 bb.0: @@ -292,19 +296,21 @@ body: | ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00 ; GFX10-NEXT: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY1]](s32), [[C1]] + ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[FCMP]](s1) ; GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.1: ; GFX10-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %39(s1), %bb.5 - ; GFX10-NEXT: [[PHI1:%[0-9]+]]:_(s32) = G_PHI %15(s32), %bb.5, [[C]](s32), %bb.0 - ; GFX10-NEXT: [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.5 - ; GFX10-NEXT: [[PHI3:%[0-9]+]]:sreg_32(s1) = G_PHI [[FCMP]](s1), %bb.0, %19(s1), %bb.5 - ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1) + ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %42(s1), %bb.5 + ; GFX10-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[COPY8]](s1), %bb.0, %39(s1), %bb.5 + ; GFX10-NEXT: [[PHI2:%[0-9]+]]:_(s32) = G_PHI %15(s32), %bb.5, [[C]](s32), %bb.0 + ; GFX10-NEXT: [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.5 + ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1) + ; GFX10-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1) ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1000 - ; GFX10-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(sle), [[PHI2]](s32), [[C3]] + ; GFX10-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(sle), [[PHI3]](s32), [[C3]] ; GFX10-NEXT: G_BRCOND [[ICMP]](s1), %bb.4 ; GFX10-NEXT: G_BR %bb.2 ; GFX10-NEXT: {{ $}} @@ -336,26 +342,27 @@ body: | ; GFX10-NEXT: successors: %bb.6(0x04000000), %bb.1(0x7c000000) ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[C8:%[0-9]+]]:_(s1) = G_CONSTANT i1 true - ; GFX10-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[PHI3]], [[C8]] - ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[XOR1]](s1) - ; GFX10-NEXT: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI2]](s32) + ; GFX10-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[COPY10]], [[C8]] + ; GFX10-NEXT: [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[XOR1]](s1) + ; GFX10-NEXT: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI3]](s32) ; GFX10-NEXT: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]] ; GFX10-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 - ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI2]], [[C9]] - ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[FCMP1]](s1), [[PHI1]](s32) - ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY9]](s1), implicit-def $scc + ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI3]], [[C9]] + ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[FCMP1]](s1), [[PHI2]](s32) + ; GFX10-NEXT: [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[XOR1]](s1) + ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY11]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc ; GFX10-NEXT: SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.6 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.6: ; GFX10-NEXT: [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.5 - ; GFX10-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1) + ; GFX10-NEXT: [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI5]](s32) ; GFX10-NEXT: [[C10:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00 ; GFX10-NEXT: [[C11:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00 - ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY10]](s1), [[C11]], [[C10]] + ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY13]](s1), [[C11]], [[C10]] ; GFX10-NEXT: G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32)) ; GFX10-NEXT: SI_RETURN bb.0: @@ -475,6 +482,7 @@ body: | ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s1) = G_TRUNC [[AND1]](s32) ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s1) = G_CONSTANT i1 true ; GFX10-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[TRUNC1]], [[C5]] + ; GFX10-NEXT: [[COPY3:%[0-9]+]]:sreg_32(s1) = COPY [[C5]](s1) ; GFX10-NEXT: G_BRCOND [[XOR]](s1), %bb.2 ; GFX10-NEXT: G_BR %bb.1 ; GFX10-NEXT: {{ $}} @@ -487,9 +495,10 @@ body: | ; GFX10-NEXT: bb.2: ; GFX10-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI:%[0-9]+]]:_(s32) = G_PHI %30(s32), %bb.4, [[DEF]](s32), %bb.0 - ; GFX10-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = G_PHI %32(s1), %bb.4, [[C5]](s1), %bb.0 - ; GFX10-NEXT: G_BRCOND [[PHI1]](s1), %bb.5 + ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY3]](s1), %bb.0, %58(s1), %bb.4 + ; GFX10-NEXT: [[PHI1:%[0-9]+]]:_(s32) = G_PHI %30(s32), %bb.4, [[DEF]](s32), %bb.0 + ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1) + ; GFX10-NEXT: G_BRCOND [[COPY4]](s1), %bb.5 ; GFX10-NEXT: G_BR %bb.6 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.3: @@ -517,6 +526,7 @@ body: | ; GFX10-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[PHI5]](s32), [[AMDGPU_BUFFER_LOAD]] ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s1) = G_OR [[ICMP]], [[ICMP2]] ; GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s1) + ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[C10]](s1) ; GFX10-NEXT: G_BR %bb.2 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.5: @@ -527,7 +537,7 @@ body: | ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[C11]] ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.6: - ; GFX10-NEXT: [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[PHI]](s32), %bb.2, [[OR2]](s32), %bb.5 + ; GFX10-NEXT: [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[PHI1]](s32), %bb.2, [[OR2]](s32), %bb.5 ; GFX10-NEXT: [[UV:%[0-9]+]]:_(<4 x s32>), [[UV1:%[0-9]+]]:_(<4 x s32>) = G_UNMERGE_VALUES [[LOAD]](<8 x s32>) ; GFX10-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[COPY2]], [[COPY1]] ; GFX10-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 2 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll index e9df20f..49c2326 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -amdgpu-global-isel-risky-select -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s +; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s ; This file contains various tests that have divergent i1s used outside of ; the loop. These are lane masks is sgpr and need to have correct value in @@ -137,28 +137,24 @@ define void @divergent_i1_xor_used_outside_loop(float %val, float %pre.cond.val, ; GFX10-LABEL: divergent_i1_xor_used_outside_loop: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, 1.0, v1 -; GFX10-NEXT: s_mov_b32 s5, 0 +; GFX10-NEXT: s_mov_b32 s4, 0 +; GFX10-NEXT: v_cmp_lt_f32_e64 s5, 1.0, v1 +; GFX10-NEXT: v_mov_b32_e32 v1, s4 ; GFX10-NEXT: ; implicit-def: $sgpr6 -; GFX10-NEXT: v_mov_b32_e32 v1, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo ; GFX10-NEXT: .LBB2_1: ; %loop ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX10-NEXT: v_and_b32_e32 v4, 1, v4 -; GFX10-NEXT: v_cvt_f32_u32_e32 v5, v1 +; GFX10-NEXT: v_cvt_f32_u32_e32 v4, v1 +; GFX10-NEXT: s_xor_b32 s5, s5, -1 ; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1 -; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4 -; GFX10-NEXT: v_cmp_gt_f32_e64 s4, v5, v0 -; GFX10-NEXT: s_xor_b32 s7, vcc_lo, -1 -; GFX10-NEXT: s_or_b32 s5, s4, s5 -; GFX10-NEXT: v_mov_b32_e32 v4, s7 -; GFX10-NEXT: s_andn2_b32 s4, s6, exec_lo -; GFX10-NEXT: s_and_b32 s6, exec_lo, s7 -; GFX10-NEXT: s_or_b32 s6, s4, s6 -; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5 +; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v0 +; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4 +; GFX10-NEXT: s_andn2_b32 s6, s6, exec_lo +; GFX10-NEXT: s_and_b32 s7, exec_lo, s5 +; GFX10-NEXT: s_or_b32 s6, s6, s7 +; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4 ; GFX10-NEXT: s_cbranch_execnz .LBB2_1 ; GFX10-NEXT: ; %bb.2: ; %exit -; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s6 ; GFX10-NEXT: flat_store_dword v[2:3], v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) @@ -197,7 +193,7 @@ define void @divergent_i1_xor_used_outside_loop_larger_loop_body(i32 %num.elts, ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 ; GFX10-NEXT: s_mov_b32 s5, 0 -; GFX10-NEXT: s_mov_b32 s6, 1 +; GFX10-NEXT: s_mov_b32 s6, -1 ; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo ; GFX10-NEXT: s_cbranch_execz .LBB3_6 ; GFX10-NEXT: ; %bb.1: ; %loop.start.preheader @@ -332,7 +328,7 @@ define void @divergent_i1_icmp_used_outside_loop(i32 %v0, i32 %v1, ptr addrspace ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s7 ; GFX10-NEXT: v_cmp_ne_u32_e64 s4, v1, v4 -; GFX10-NEXT: s_mov_b32 s7, 1 +; GFX10-NEXT: s_mov_b32 s7, -1 ; GFX10-NEXT: ; implicit-def: $vgpr5 ; GFX10-NEXT: s_and_saveexec_b32 s8, s4 ; GFX10-NEXT: s_cbranch_execz .LBB4_1 @@ -410,7 +406,7 @@ define amdgpu_ps void @divergent_i1_freeze_used_outside_loop(i32 %n, ptr addrspa ; GFX10-LABEL: divergent_i1_freeze_used_outside_loop: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_mov_b32 s0, 0 -; GFX10-NEXT: s_mov_b32 s3, 1 +; GFX10-NEXT: s_mov_b32 s3, -1 ; GFX10-NEXT: v_mov_b32_e32 v5, s0 ; GFX10-NEXT: ; implicit-def: $sgpr1 ; GFX10-NEXT: ; implicit-def: $sgpr2 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.mir index ace9bec..206c0ad 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.mir @@ -175,14 +175,15 @@ body: | ; GFX10-NEXT: [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc ; GFX10-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY13]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_3:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc + ; GFX10-NEXT: [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[COPY7]](s1) ; GFX10-NEXT: G_BRCOND [[ICMP1]](s1), %bb.1 ; GFX10-NEXT: G_BR %bb.4 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.4: - ; GFX10-NEXT: [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[COPY7]](s1) + ; GFX10-NEXT: [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[COPY14]](s1) ; GFX10-NEXT: [[C6:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00 ; GFX10-NEXT: [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00 - ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY14]](s1), [[C7]], [[C6]] + ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY15]](s1), [[C7]], [[C6]] ; GFX10-NEXT: G_STORE [[SELECT]](s32), [[MV1]](p0) :: (store (s32)) ; GFX10-NEXT: SI_RETURN bb.0: @@ -255,37 +256,40 @@ body: | ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00 ; GFX10-NEXT: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY1]](s32), [[C1]] + ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[FCMP]](s1) ; GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.1: ; GFX10-NEXT: successors: %bb.2(0x04000000), %bb.1(0x7c000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %24(s1), %bb.1 - ; GFX10-NEXT: [[PHI1:%[0-9]+]]:_(s32) = G_PHI %9(s32), %bb.1, [[C]](s32), %bb.0 - ; GFX10-NEXT: [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %11(s32), %bb.1 - ; GFX10-NEXT: [[PHI3:%[0-9]+]]:sreg_32(s1) = G_PHI [[FCMP]](s1), %bb.0, %13(s1), %bb.1 - ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1) + ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %27(s1), %bb.1 + ; GFX10-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[COPY4]](s1), %bb.0, %24(s1), %bb.1 + ; GFX10-NEXT: [[PHI2:%[0-9]+]]:_(s32) = G_PHI %9(s32), %bb.1, [[C]](s32), %bb.0 + ; GFX10-NEXT: [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %11(s32), %bb.1 + ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1) + ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1) ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true - ; GFX10-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[PHI3]], [[C2]] - ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1) - ; GFX10-NEXT: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI2]](s32) + ; GFX10-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[COPY6]], [[C2]] + ; GFX10-NEXT: [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1) + ; GFX10-NEXT: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI3]](s32) ; GFX10-NEXT: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]] ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 - ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI2]], [[C3]] - ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[FCMP1]](s1), [[PHI1]](s32) - ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY4]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY5]](s1), implicit-def $scc + ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI3]], [[C3]] + ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[FCMP1]](s1), [[PHI2]](s32) + ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1) + ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY5]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY7]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc ; GFX10-NEXT: SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.2 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.2: ; GFX10-NEXT: [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.1 - ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1) + ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32) ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00 ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00 - ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY6]](s1), [[C5]], [[C4]] + ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY9]](s1), [[C5]], [[C4]] ; GFX10-NEXT: G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32)) ; GFX10-NEXT: SI_RETURN bb.0: @@ -349,7 +353,8 @@ body: | ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX10-NEXT: [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]] ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true - ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[C1]](s1) + ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[C1]](s1) + ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[COPY5]](s1) ; GFX10-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.1 ; GFX10-NEXT: {{ $}} @@ -365,26 +370,26 @@ body: | ; GFX10-NEXT: bb.2: ; GFX10-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[C1]](s1), %bb.0, %39(s1), %bb.8 - ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI]](s1) + ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[COPY5]](s1), %bb.0, %40(s1), %bb.8 + ; GFX10-NEXT: [[COPY7:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32) - ; GFX10-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY6]](s1), %bb.6, implicit-def $exec, implicit-def $scc, implicit $exec + ; GFX10-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY7]](s1), %bb.6, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.5 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.3: ; GFX10-NEXT: successors: %bb.4(0x40000000), %bb.7(0x40000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF3]](s1), %bb.1, %72(s1), %bb.7 - ; GFX10-NEXT: [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[DEF2]](s1), %bb.1, %61(s1), %bb.7 - ; GFX10-NEXT: [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.1, %48(s1), %bb.7 + ; GFX10-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF3]](s1), %bb.1, %73(s1), %bb.7 + ; GFX10-NEXT: [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[DEF2]](s1), %bb.1, %62(s1), %bb.7 + ; GFX10-NEXT: [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.1, %49(s1), %bb.7 ; GFX10-NEXT: [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[C2]](s32), %bb.1, %17(s32), %bb.7 ; GFX10-NEXT: [[PHI5:%[0-9]+]]:_(s32) = G_PHI %19(s32), %bb.7, [[C2]](s32), %bb.1 - ; GFX10-NEXT: [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1) - ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI2]](s1) - ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1) + ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1) + ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[PHI2]](s1) + ; GFX10-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1) ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 true - ; GFX10-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1) ; GFX10-NEXT: [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1) + ; GFX10-NEXT: [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1) ; GFX10-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI5]](s32) ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2 ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C4]](s32) @@ -392,14 +397,14 @@ body: | ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1) ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX10-NEXT: [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C5]] - ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY11]](s1), implicit-def $scc + ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY12]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc - ; GFX10-NEXT: [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1) - ; GFX10-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc + ; GFX10-NEXT: [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1) + ; GFX10-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY11]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc - ; GFX10-NEXT: [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_1]](s1) + ; GFX10-NEXT: [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_1]](s1) ; GFX10-NEXT: [[SI_IF2:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.4 ; GFX10-NEXT: {{ $}} @@ -407,16 +412,16 @@ body: | ; GFX10-NEXT: successors: %bb.7(0x80000000) ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[C6:%[0-9]+]]:_(s1) = G_CONSTANT i1 false - ; GFX10-NEXT: [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[C6]](s1) + ; GFX10-NEXT: [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[C6]](s1) ; GFX10-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI5]], [[C7]] ; GFX10-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[PHI5]](s32), [[COPY]] - ; GFX10-NEXT: [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1) - ; GFX10-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY12]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY14]](s1), implicit-def $scc + ; GFX10-NEXT: [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1) + ; GFX10-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY13]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY15]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc - ; GFX10-NEXT: [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY13]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY15]](s1), implicit-def $scc + ; GFX10-NEXT: [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY14]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY16]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_3:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc ; GFX10-NEXT: G_BR %bb.7 ; GFX10-NEXT: {{ $}} @@ -436,15 +441,15 @@ body: | ; GFX10-NEXT: [[PHI6:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_1]](s1), %bb.3, [[S_OR_B32_3]](s1), %bb.4 ; GFX10-NEXT: [[PHI7:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.3, [[S_OR_B32_2]](s1), %bb.4 ; GFX10-NEXT: [[PHI8:%[0-9]+]]:_(s32) = G_PHI [[ADD]](s32), %bb.4, [[DEF]](s32), %bb.3 - ; GFX10-NEXT: [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[PHI6]](s1) - ; GFX10-NEXT: [[COPY17:%[0-9]+]]:sreg_32(s1) = COPY [[PHI7]](s1) + ; GFX10-NEXT: [[COPY17:%[0-9]+]]:sreg_32(s1) = COPY [[PHI6]](s1) + ; GFX10-NEXT: [[COPY18:%[0-9]+]]:sreg_32(s1) = COPY [[PHI7]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF2]](s32) ; GFX10-NEXT: [[C9:%[0-9]+]]:_(s1) = G_CONSTANT i1 true - ; GFX10-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[COPY17]], [[C9]] - ; GFX10-NEXT: [[COPY18:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1) - ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[COPY16]](s1), [[PHI4]](s32) - ; GFX10-NEXT: [[S_ANDN2_B32_4:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY7]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_4:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY18]](s1), implicit-def $scc + ; GFX10-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[COPY18]], [[C9]] + ; GFX10-NEXT: [[COPY19:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1) + ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[COPY17]](s1), [[PHI4]](s32) + ; GFX10-NEXT: [[S_ANDN2_B32_4:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_4:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY19]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_4:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_4]](s1), [[S_AND_B32_4]](s1), implicit-def $scc ; GFX10-NEXT: SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.8 @@ -453,11 +458,11 @@ body: | ; GFX10-NEXT: successors: %bb.2(0x80000000) ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[PHI9:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.7 - ; GFX10-NEXT: [[COPY19:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_4]](s1) - ; GFX10-NEXT: [[COPY20:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[COPY19]](s1) + ; GFX10-NEXT: [[COPY20:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_4]](s1) + ; GFX10-NEXT: [[COPY21:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[COPY20]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI9]](s32) - ; GFX10-NEXT: [[S_ANDN2_B32_5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY5]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY20]](s1), implicit-def $scc + ; GFX10-NEXT: [[S_ANDN2_B32_5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY21]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_5]](s1), [[S_AND_B32_5]](s1), implicit-def $scc ; GFX10-NEXT: G_BR %bb.2 bb.0: @@ -574,7 +579,7 @@ body: | ; GFX10-NEXT: bb.1: ; GFX10-NEXT: successors: %bb.2(0x80000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[DEF1]](s1), %bb.0, %38(s1), %bb.6 + ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[DEF1]](s1), %bb.0, %39(s1), %bb.6 ; GFX10-NEXT: [[PHI1:%[0-9]+]]:_(s32) = G_PHI %11(s32), %bb.6, [[C]](s32), %bb.0 ; GFX10-NEXT: [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %13(s32), %bb.6 ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI]](s1) @@ -600,9 +605,10 @@ body: | ; GFX10-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000) ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true - ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[C2]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32) ; GFX10-NEXT: [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[PHI2]] + ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[C2]](s1) + ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[COPY8]](s1) ; GFX10-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.6, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.5 ; GFX10-NEXT: {{ $}} @@ -610,21 +616,21 @@ body: | ; GFX10-NEXT: successors: %bb.6(0x80000000) ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 false - ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1) + ; GFX10-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1) ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI2]], [[C4]] - ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY9]](s1), implicit-def $scc + ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.6: ; GFX10-NEXT: successors: %bb.7(0x04000000), %bb.1(0x7c000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[C2]](s1), %bb.4, [[S_OR_B32_]](s1), %bb.5 + ; GFX10-NEXT: [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[COPY8]](s1), %bb.4, [[S_OR_B32_]](s1), %bb.5 ; GFX10-NEXT: [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[ADD]](s32), %bb.5, [[DEF]](s32), %bb.4 - ; GFX10-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1) + ; GFX10-NEXT: [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF1]](s32) - ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[COPY10]](s1), [[PHI1]](s32) + ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[COPY11]](s1), [[PHI1]](s32) ; GFX10-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc ; GFX10-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY7]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc @@ -636,9 +642,9 @@ body: | ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.6 ; GFX10-NEXT: [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[PHI2]](s32), %bb.6 - ; GFX10-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[S_OR_B32_1]](s1) + ; GFX10-NEXT: [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[S_OR_B32_1]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI5]](s32) - ; GFX10-NEXT: [[SI_IF2:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY11]](s1), %bb.9, implicit-def $exec, implicit-def $scc, implicit $exec + ; GFX10-NEXT: [[SI_IF2:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY12]](s1), %bb.9, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.8 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.8: @@ -751,26 +757,27 @@ body: | ; GFX10-NEXT: [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32) ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true + ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[C1]](s1) ; GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF ; GFX10-NEXT: [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.1: ; GFX10-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %53(s1), %bb.3 - ; GFX10-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %42(s1), %bb.3 - ; GFX10-NEXT: [[PHI2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[C1]](s1), %bb.0, %32(s1), %bb.3 + ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %54(s1), %bb.3 + ; GFX10-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %43(s1), %bb.3 + ; GFX10-NEXT: [[PHI2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[COPY5]](s1), %bb.0, %33(s1), %bb.3 ; GFX10-NEXT: [[PHI3:%[0-9]+]]:_(s32) = G_PHI %10(s32), %bb.3, [[C]](s32), %bb.0 ; GFX10-NEXT: [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %12(s32), %bb.3 - ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1) - ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1) - ; GFX10-NEXT: [[COPY7:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI2]](s1) - ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[COPY7]](s1) - ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY8]](s1), implicit-def $scc + ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1) + ; GFX10-NEXT: [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1) + ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI2]](s1) + ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[COPY8]](s1) + ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY7]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY9]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc - ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1) - ; GFX10-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY7]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec + ; GFX10-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1) + ; GFX10-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY8]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.2 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.2: @@ -783,10 +790,10 @@ body: | ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1) ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX10-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[LOAD]](s32), [[C3]] - ; GFX10-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP]](s1) + ; GFX10-NEXT: [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP]](s1) ; GFX10-NEXT: [[DEF2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = IMPLICIT_DEF - ; GFX10-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc + ; GFX10-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY11]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.3: @@ -794,32 +801,32 @@ body: | ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, [[S_OR_B32_1]](s1), %bb.2 ; GFX10-NEXT: [[PHI6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[PHI2]](s1), %bb.1, [[DEF2]](s1), %bb.2 - ; GFX10-NEXT: [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1) - ; GFX10-NEXT: [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI6]](s1) + ; GFX10-NEXT: [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1) + ; GFX10-NEXT: [[COPY13:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI6]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32) - ; GFX10-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE [[COPY11]] - ; GFX10-NEXT: [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[FREEZE]](s1) - ; GFX10-NEXT: [[COPY14:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[FREEZE]](s1) + ; GFX10-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE [[COPY12]] + ; GFX10-NEXT: [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[FREEZE]](s1) + ; GFX10-NEXT: [[COPY15:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[FREEZE]](s1) ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI4]], [[C4]] ; GFX10-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[PHI4]](s32), [[COPY]] ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[ICMP1]](s1), [[PHI3]](s32) - ; GFX10-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY12]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY14]](s1), implicit-def $scc + ; GFX10-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY13]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY15]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc - ; GFX10-NEXT: [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY5]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY13]](s1), implicit-def $scc + ; GFX10-NEXT: [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY14]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_3:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc ; GFX10-NEXT: SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.4 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.4: ; GFX10-NEXT: [[PHI7:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.3 - ; GFX10-NEXT: [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_3]](s1) + ; GFX10-NEXT: [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_3]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI7]](s32) ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00 ; GFX10-NEXT: [[C6:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00 - ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY15]](s1), [[C6]], [[C5]] + ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY16]](s1), [[C6]], [[C5]] ; GFX10-NEXT: G_STORE [[SELECT]](s32), [[MV1]](p0) :: (store (s32)) ; GFX10-NEXT: S_ENDPGM 0 bb.0: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll index 609fff5..1698f84 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -amdgpu-global-isel-risky-select -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s +; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s ; Simples case, if - then, that requires lane mask merging, ; %phi lane mask will hold %val_A at %A. Lanes that are active in %B @@ -43,13 +43,12 @@ define amdgpu_ps void @divergent_i1_phi_if_else(ptr addrspace(1) %out, i32 %tid, ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_and_b32 s0, 1, s0 ; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3 -; GFX10-NEXT: v_cmp_ne_u32_e64 s2, 0, s0 -; GFX10-NEXT: ; implicit-def: $sgpr0 +; GFX10-NEXT: v_cmp_ne_u32_e64 s0, 0, s0 ; GFX10-NEXT: s_and_saveexec_b32 s1, vcc_lo ; GFX10-NEXT: s_xor_b32 s1, exec_lo, s1 ; GFX10-NEXT: ; %bb.1: ; %B ; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 2, v2 -; GFX10-NEXT: s_andn2_b32 s0, s2, exec_lo +; GFX10-NEXT: s_andn2_b32 s0, s0, exec_lo ; GFX10-NEXT: ; implicit-def: $vgpr2 ; GFX10-NEXT: s_and_b32 s2, exec_lo, vcc_lo ; GFX10-NEXT: s_or_b32 s0, s0, s2 @@ -211,7 +210,7 @@ define amdgpu_cs void @loop_with_2breaks(ptr addrspace(1) %x, ptr addrspace(1) % ; GFX10-NEXT: ; in Loop: Header=BB3_3 Depth=1 ; GFX10-NEXT: v_add_co_u32 v9, vcc_lo, v4, v7 ; GFX10-NEXT: v_add_co_ci_u32_e32 v10, vcc_lo, v5, v8, vcc_lo -; GFX10-NEXT: s_mov_b32 s4, 1 +; GFX10-NEXT: s_mov_b32 s4, -1 ; GFX10-NEXT: global_load_dword v9, v[9:10], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v9 @@ -308,7 +307,7 @@ define amdgpu_cs void @loop_with_3breaks(ptr addrspace(1) %x, ptr addrspace(1) % ; GFX10-NEXT: ; in Loop: Header=BB4_4 Depth=1 ; GFX10-NEXT: v_add_co_u32 v11, vcc_lo, v4, v9 ; GFX10-NEXT: v_add_co_ci_u32_e32 v12, vcc_lo, v5, v10, vcc_lo -; GFX10-NEXT: s_mov_b32 s4, 1 +; GFX10-NEXT: s_mov_b32 s4, -1 ; GFX10-NEXT: global_load_dword v11, v[11:12], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v11 @@ -318,7 +317,7 @@ define amdgpu_cs void @loop_with_3breaks(ptr addrspace(1) %x, ptr addrspace(1) % ; GFX10-NEXT: ; in Loop: Header=BB4_4 Depth=1 ; GFX10-NEXT: v_add_co_u32 v11, vcc_lo, v6, v9 ; GFX10-NEXT: v_add_co_ci_u32_e32 v12, vcc_lo, v7, v10, vcc_lo -; GFX10-NEXT: s_mov_b32 s5, 1 +; GFX10-NEXT: s_mov_b32 s5, -1 ; GFX10-NEXT: global_load_dword v11, v[11:12], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v11 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.mir index df5505e..8197b07 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.mir @@ -18,9 +18,10 @@ body: | ; GFX10-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3 ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 6 ; GFX10-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY2]](s32), [[C]] - ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP]](s1) ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX10-NEXT: [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), [[COPY3]](s32), [[C1]] + ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP]](s1) + ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[COPY4]](s1) ; GFX10-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.1 ; GFX10-NEXT: {{ $}} @@ -29,18 +30,18 @@ body: | ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; GFX10-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY2]](s32), [[C2]] - ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1) - ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY4]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY5]](s1), implicit-def $scc + ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1) + ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY5]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY6]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.2: - ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[ICMP]](s1), %bb.0, [[S_OR_B32_]](s1), %bb.1 - ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1) + ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY4]](s1), %bb.0, [[S_OR_B32_]](s1), %bb.1 + ; GFX10-NEXT: [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32) ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2 ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 - ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY6]](s1), [[C4]], [[C3]] + ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY7]](s1), [[C4]], [[C3]] ; GFX10-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1) ; GFX10-NEXT: S_ENDPGM 0 bb.0: @@ -91,18 +92,20 @@ body: | ; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2 ; GFX10-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3 ; GFX10-NEXT: [[DEF:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF - ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[DEF]](s1) ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX10-NEXT: [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[COPY3]](s32), [[C]] + ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[DEF]](s1) + ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[COPY4]](s1) ; GFX10-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.3 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.1: ; GFX10-NEXT: successors: %bb.2(0x40000000), %bb.4(0x40000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %19(s1), %bb.3 - ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1) - ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[COPY5]](s1) + ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY4]](s1), %bb.0, %20(s1), %bb.3 + ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1) + ; GFX10-NEXT: [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[COPY6]](s1) + ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[COPY7]](s1) ; GFX10-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_ELSE [[SI_IF]](s32), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.2 ; GFX10-NEXT: {{ $}} @@ -111,9 +114,9 @@ body: | ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; GFX10-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY2]](s32), [[C1]] - ; GFX10-NEXT: [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP1]](s1) - ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY7]](s1), implicit-def $scc + ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP1]](s1) + ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY9]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc ; GFX10-NEXT: G_BR %bb.4 ; GFX10-NEXT: {{ $}} @@ -122,19 +125,19 @@ body: | ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2 ; GFX10-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY2]](s32), [[C2]] - ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1) - ; GFX10-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY4]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY8]](s1), implicit-def $scc + ; GFX10-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1) + ; GFX10-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY5]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc ; GFX10-NEXT: G_BR %bb.1 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.4: - ; GFX10-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[COPY5]](s1), %bb.1, [[S_OR_B32_]](s1), %bb.2 - ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1) + ; GFX10-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[COPY7]](s1), %bb.1, [[S_OR_B32_]](s1), %bb.2 + ; GFX10-NEXT: [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_ELSE]](s32) ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2 - ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY9]](s1), [[C3]], [[C4]] + ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY11]](s1), [[C3]], [[C4]] ; GFX10-NEXT: G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1) ; GFX10-NEXT: S_ENDPGM 0 bb.0: @@ -368,13 +371,14 @@ body: | ; GFX10-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000) ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true - ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C4]](s1) ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2 ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C5]](s32) ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV2]], [[SHL1]](s64) ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1) ; GFX10-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX10-NEXT: [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[C6]] + ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C4]](s1) + ; GFX10-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[COPY9]](s1) ; GFX10-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.4 ; GFX10-NEXT: {{ $}} @@ -383,9 +387,9 @@ body: | ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %47(s1), %bb.5 ; GFX10-NEXT: [[PHI4:%[0-9]+]]:_(s32) = G_PHI %32(s32), %bb.5, [[DEF]](s32), %bb.1 - ; GFX10-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1) + ; GFX10-NEXT: [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32) - ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[COPY10]](s1), [[PHI1]](s32) + ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[COPY11]](s1), [[PHI1]](s32) ; GFX10-NEXT: SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.6 ; GFX10-NEXT: {{ $}} @@ -402,21 +406,21 @@ body: | ; GFX10-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI2]], [[C8]] ; GFX10-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 100 ; GFX10-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI2]](s32), [[C9]] - ; GFX10-NEXT: [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1) - ; GFX10-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY11]](s1), implicit-def $scc + ; GFX10-NEXT: [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1) + ; GFX10-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY12]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.5: ; GFX10-NEXT: successors: %bb.3(0x80000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[C4]](s1), %bb.2, [[S_OR_B32_1]](s1), %bb.4 + ; GFX10-NEXT: [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[COPY9]](s1), %bb.2, [[S_OR_B32_1]](s1), %bb.4 ; GFX10-NEXT: [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[ADD1]](s32), %bb.4, [[DEF]](s32), %bb.2 - ; GFX10-NEXT: [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1) - ; GFX10-NEXT: [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[COPY12]](s1) + ; GFX10-NEXT: [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1) + ; GFX10-NEXT: [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[COPY13]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF1]](s32) ; GFX10-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY13]](s1), implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY14]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc ; GFX10-NEXT: G_BR %bb.3 ; GFX10-NEXT: {{ $}} @@ -560,13 +564,14 @@ body: | ; GFX10-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000) ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true - ; GFX10-NEXT: [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[C4]](s1) ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2 ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C5]](s32) ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV2]], [[SHL1]](s64) ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1) ; GFX10-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX10-NEXT: [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[C6]] + ; GFX10-NEXT: [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[C4]](s1) + ; GFX10-NEXT: [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[COPY11]](s1) ; GFX10-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.4 ; GFX10-NEXT: {{ $}} @@ -575,9 +580,9 @@ body: | ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %60(s1), %bb.5 ; GFX10-NEXT: [[PHI4:%[0-9]+]]:_(s32) = G_PHI %35(s32), %bb.5, [[DEF]](s32), %bb.1 - ; GFX10-NEXT: [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1) + ; GFX10-NEXT: [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32) - ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[COPY12]](s1), [[PHI1]](s32) + ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[COPY13]](s1), [[PHI1]](s32) ; GFX10-NEXT: SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.8 ; GFX10-NEXT: {{ $}} @@ -585,26 +590,27 @@ body: | ; GFX10-NEXT: successors: %bb.6(0x40000000), %bb.7(0x40000000) ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[C7:%[0-9]+]]:_(s1) = G_CONSTANT i1 true - ; GFX10-NEXT: [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[C7]](s1) ; GFX10-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 2 ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C8]](s32) ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV3]], [[SHL2]](s64) ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s32), addrspace 1) ; GFX10-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX10-NEXT: [[ICMP2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD2]](s32), [[C9]] + ; GFX10-NEXT: [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[C7]](s1) + ; GFX10-NEXT: [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[COPY14]](s1) ; GFX10-NEXT: [[SI_IF2:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP2]](s1), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.6 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.5: ; GFX10-NEXT: successors: %bb.3(0x80000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[C4]](s1), %bb.2, %71(s1), %bb.7 + ; GFX10-NEXT: [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[COPY11]](s1), %bb.2, %72(s1), %bb.7 ; GFX10-NEXT: [[PHI6:%[0-9]+]]:_(s32) = G_PHI %46(s32), %bb.7, [[DEF]](s32), %bb.2 - ; GFX10-NEXT: [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1) - ; GFX10-NEXT: [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[COPY14]](s1) + ; GFX10-NEXT: [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1) + ; GFX10-NEXT: [[COPY17:%[0-9]+]]:sreg_32(s1) = COPY [[COPY16]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF1]](s32) ; GFX10-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY15]](s1), implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY17]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc ; GFX10-NEXT: G_BR %bb.3 ; GFX10-NEXT: {{ $}} @@ -621,21 +627,21 @@ body: | ; GFX10-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI2]], [[C11]] ; GFX10-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 100 ; GFX10-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI2]](s32), [[C12]] - ; GFX10-NEXT: [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP3]](s1) - ; GFX10-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY13]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY16]](s1), implicit-def $scc + ; GFX10-NEXT: [[COPY18:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP3]](s1) + ; GFX10-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY15]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY18]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.7: ; GFX10-NEXT: successors: %bb.5(0x80000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI7:%[0-9]+]]:sreg_32(s1) = PHI [[C7]](s1), %bb.4, [[S_OR_B32_2]](s1), %bb.6 + ; GFX10-NEXT: [[PHI7:%[0-9]+]]:sreg_32(s1) = PHI [[COPY14]](s1), %bb.4, [[S_OR_B32_2]](s1), %bb.6 ; GFX10-NEXT: [[PHI8:%[0-9]+]]:_(s32) = G_PHI [[ADD1]](s32), %bb.6, [[DEF]](s32), %bb.4 - ; GFX10-NEXT: [[COPY17:%[0-9]+]]:sreg_32(s1) = COPY [[PHI7]](s1) - ; GFX10-NEXT: [[COPY18:%[0-9]+]]:sreg_32(s1) = COPY [[COPY17]](s1) + ; GFX10-NEXT: [[COPY19:%[0-9]+]]:sreg_32(s1) = COPY [[PHI7]](s1) + ; GFX10-NEXT: [[COPY20:%[0-9]+]]:sreg_32(s1) = COPY [[COPY19]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF2]](s32) - ; GFX10-NEXT: [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY11]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY18]](s1), implicit-def $scc + ; GFX10-NEXT: [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY12]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY20]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_3:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc ; GFX10-NEXT: G_BR %bb.5 ; GFX10-NEXT: {{ $}} @@ -970,6 +976,7 @@ body: | ; GFX10-NEXT: [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF ; GFX10-NEXT: [[DEF2:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF ; GFX10-NEXT: [[DEF3:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF + ; GFX10-NEXT: [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP]](s1) ; GFX10-NEXT: G_BR %bb.7 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.1: @@ -982,19 +989,19 @@ body: | ; GFX10-NEXT: bb.2: ; GFX10-NEXT: successors: %bb.4(0x40000000), %bb.7(0x40000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI %67(s1), %bb.6, %70(s1), %bb.7 + ; GFX10-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI %67(s1), %bb.6, %71(s1), %bb.7 ; GFX10-NEXT: [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI %49(s1), %bb.6, %48(s1), %bb.7 ; GFX10-NEXT: [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI %35(s1), %bb.6, %34(s1), %bb.7 - ; GFX10-NEXT: [[COPY7:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI]](s1) - ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1) - ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[PHI2]](s1) - ; GFX10-NEXT: [[COPY10:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[COPY9]](s1) + ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI]](s1) + ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1) + ; GFX10-NEXT: [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[PHI2]](s1) + ; GFX10-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[COPY10]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %15(s32) - ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[COPY8]](s1), %17(s32) - ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY7]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc + ; GFX10-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[COPY9]](s1), %17(s32) + ; GFX10-NEXT: [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY11]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc - ; GFX10-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[S_OR_B32_]](s1) + ; GFX10-NEXT: [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[S_OR_B32_]](s1) ; GFX10-NEXT: SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.4 ; GFX10-NEXT: {{ $}} @@ -1011,28 +1018,28 @@ body: | ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INTRINSIC_CONVERGENT]](s32) ; GFX10-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY5]](s32), [[COPY]] - ; GFX10-NEXT: [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1) + ; GFX10-NEXT: [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1) ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true - ; GFX10-NEXT: [[COPY13:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[C2]](s1) + ; GFX10-NEXT: [[COPY14:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[C2]](s1) ; GFX10-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP]], [[C2]] ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s1) = G_OR [[ICMP2]], [[XOR]] ; GFX10-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[OR]](s1), %25(s32) ; GFX10-NEXT: [[DEF4:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF ; GFX10-NEXT: [[DEF5:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF ; GFX10-NEXT: [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 %63(s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY12]](s1), implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY13]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc - ; GFX10-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY11]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY13]](s1), implicit-def $scc + ; GFX10-NEXT: [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY12]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY14]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc ; GFX10-NEXT: SI_LOOP [[INTRINSIC_CONVERGENT2]](s32), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.5 ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: bb.5: ; GFX10-NEXT: [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT2]](s32), %bb.4 - ; GFX10-NEXT: [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_1]](s1) + ; GFX10-NEXT: [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_1]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32) - ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY14]](s1), [[COPY3]], [[COPY2]] + ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY15]](s1), [[COPY3]], [[COPY2]] ; GFX10-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[SELECT]](s32) ; GFX10-NEXT: $sgpr0 = COPY [[INTRINSIC_CONVERGENT3]](s32) ; GFX10-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0 @@ -1042,14 +1049,14 @@ body: | ; GFX10-NEXT: {{ $}} ; GFX10-NEXT: [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT1]](s32), %bb.3 ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 false - ; GFX10-NEXT: [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1) ; GFX10-NEXT: [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1) + ; GFX10-NEXT: [[COPY17:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1) ; GFX10-NEXT: G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI5]](s32) ; GFX10-NEXT: [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 %42(s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY16]](s1), implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY17]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_3:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc ; GFX10-NEXT: [[S_ANDN2_B32_4:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 %56(s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_4:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY15]](s1), implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_4:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY16]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_4:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_4]](s1), [[S_AND_B32_4]](s1), implicit-def $scc ; GFX10-NEXT: [[DEF6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = IMPLICIT_DEF ; GFX10-NEXT: G_BR %bb.2 @@ -1057,27 +1064,27 @@ body: | ; GFX10-NEXT: bb.7: ; GFX10-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[PHI6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[ICMP]](s1), %bb.0, [[S_OR_B32_]](s1), %bb.2, [[S_OR_B32_2]](s1), %bb.4 + ; GFX10-NEXT: [[PHI6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[COPY7]](s1), %bb.0, [[S_OR_B32_]](s1), %bb.2, [[S_OR_B32_2]](s1), %bb.4 ; GFX10-NEXT: [[PHI7:%[0-9]+]]:sreg_32(s1) = PHI [[DEF3]](s1), %bb.0, [[PHI7]](s1), %bb.2, [[S_OR_B32_1]](s1), %bb.4 ; GFX10-NEXT: [[PHI8:%[0-9]+]]:sreg_32(s1) = PHI [[DEF2]](s1), %bb.0, [[PHI1]](s1), %bb.2, [[DEF5]](s1), %bb.4 ; GFX10-NEXT: [[PHI9:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, [[PHI2]](s1), %bb.2, [[DEF4]](s1), %bb.4 ; GFX10-NEXT: [[PHI10:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT2]](s32), %bb.4, [[PHI10]](s32), %bb.2, [[C]](s32), %bb.0 ; GFX10-NEXT: [[PHI11:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.4, [[INTRINSIC_CONVERGENT]](s32), %bb.2, [[C]](s32), %bb.0 - ; GFX10-NEXT: [[COPY17:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI6]](s1) - ; GFX10-NEXT: [[COPY18:%[0-9]+]]:sreg_32(s1) = COPY [[PHI7]](s1) - ; GFX10-NEXT: [[COPY19:%[0-9]+]]:sreg_32(s1) = COPY [[PHI8]](s1) - ; GFX10-NEXT: [[COPY20:%[0-9]+]]:sreg_32(s1) = COPY [[PHI9]](s1) + ; GFX10-NEXT: [[COPY18:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI6]](s1) + ; GFX10-NEXT: [[COPY19:%[0-9]+]]:sreg_32(s1) = COPY [[PHI7]](s1) + ; GFX10-NEXT: [[COPY20:%[0-9]+]]:sreg_32(s1) = COPY [[PHI8]](s1) + ; GFX10-NEXT: [[COPY21:%[0-9]+]]:sreg_32(s1) = COPY [[PHI9]](s1) ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true - ; GFX10-NEXT: [[COPY21:%[0-9]+]]:sreg_32(s1) = COPY [[C4]](s1) - ; GFX10-NEXT: [[S_ANDN2_B32_5:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY20]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[COPY22:%[0-9]+]]:sreg_32(s1) = COPY [[C4]](s1) + ; GFX10-NEXT: [[S_ANDN2_B32_5:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY21]](s1), $exec_lo, implicit-def $scc ; GFX10-NEXT: [[S_AND_B32_5:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY6]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_5:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_5]](s1), [[S_AND_B32_5]](s1), implicit-def $scc - ; GFX10-NEXT: [[COPY22:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_5]](s1) - ; GFX10-NEXT: [[S_ANDN2_B32_6:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY19]](s1), $exec_lo, implicit-def $scc - ; GFX10-NEXT: [[S_AND_B32_6:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY21]](s1), implicit-def $scc + ; GFX10-NEXT: [[COPY23:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_5]](s1) + ; GFX10-NEXT: [[S_ANDN2_B32_6:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY20]](s1), $exec_lo, implicit-def $scc + ; GFX10-NEXT: [[S_AND_B32_6:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY22]](s1), implicit-def $scc ; GFX10-NEXT: [[S_OR_B32_6:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_6]](s1), [[S_AND_B32_6]](s1), implicit-def $scc - ; GFX10-NEXT: [[COPY23:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_6]](s1) - ; GFX10-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY17]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec + ; GFX10-NEXT: [[COPY24:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_6]](s1) + ; GFX10-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY18]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX10-NEXT: G_BR %bb.1 bb.0: successors: %bb.7(0x80000000) diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll index 312c6a3..1855ede 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc -global-isel -amdgpu-global-isel-risky-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s +; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s define void @temporal_divergent_i1_phi(float %val, ptr %addr) { ; GFX10-LABEL: temporal_divergent_i1_phi: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll index b21e6a7..1934958 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc -global-isel -amdgpu-global-isel-risky-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s +; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s define void @temporal_divergent_i32(float %val, ptr %addr) { ; GFX10-LABEL: temporal_divergent_i32: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-phi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-phi.mir index c7d45f0..4bb9eb8 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-phi.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-phi.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=amdgcn -amdgpu-global-isel-risky-select -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=GCN +# RUN: llc -mtriple=amdgcn -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=GCN --- name: g_phi_s32_ss_sbranch @@ -322,60 +322,6 @@ body: | ... --- -name: g_phi_vcc_s1_sbranch -legalized: true -regBankSelected: true -tracksRegLiveness: true -machineFunctionInfo: {} -body: | - ; GCN-LABEL: name: g_phi_vcc_s1_sbranch - ; GCN: bb.0: - ; GCN-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) - ; GCN-NEXT: liveins: $vgpr0, $vgpr1, $sgpr2 - ; GCN-NEXT: {{ $}} - ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GCN-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2 - ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0 - ; GCN-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[COPY]], [[S_MOV_B32_]], implicit $exec - ; GCN-NEXT: S_CMP_EQ_U32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc - ; GCN-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $scc - ; GCN-NEXT: $scc = COPY [[COPY3]] - ; GCN-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc - ; GCN-NEXT: S_BRANCH %bb.2 - ; GCN-NEXT: {{ $}} - ; GCN-NEXT: bb.1: - ; GCN-NEXT: successors: %bb.2(0x80000000) - ; GCN-NEXT: {{ $}} - ; GCN-NEXT: [[V_CMP_EQ_U32_e64_1:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[COPY1]], [[S_MOV_B32_]], implicit $exec - ; GCN-NEXT: S_BRANCH %bb.2 - ; GCN-NEXT: {{ $}} - ; GCN-NEXT: bb.2: - ; GCN-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[V_CMP_EQ_U32_e64_]], %bb.0, [[V_CMP_EQ_U32_e64_1]], %bb.1 - ; GCN-NEXT: S_SETPC_B64 undef $sgpr30_sgpr31, implicit [[PHI]] - bb.0: - liveins: $vgpr0, $vgpr1, $sgpr2 - - %0:vgpr(s32) = COPY $vgpr0 - %1:vgpr(s32) = COPY $vgpr1 - %2:sgpr(s32) = COPY $sgpr2 - %3:sgpr(s32) = G_CONSTANT i32 0 - %4:vcc(s1) = G_ICMP intpred(eq), %0, %3 - %5:sgpr(s32) = G_ICMP intpred(eq), %2(s32), %3 - G_BRCOND %5, %bb.1 - G_BR %bb.2 - - bb.1: - %6:vcc(s1) = G_ICMP intpred(eq), %1, %3 - G_BR %bb.2 - - bb.2: - %7:vcc(s1) = G_PHI %4, %bb.0, %6, %bb.1 - S_SETPC_B64 undef $sgpr30_sgpr31, implicit %7 - -... - ---- name: phi_s32_ss_sbranch legalized: true regBankSelected: true diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll index 5296ad3..2f3d5d9 100644 --- a/llvm/test/CodeGen/AMDGPU/div_i128.ll +++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll @@ -2310,2860 +2310,6 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) { ret i128 %div } -define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { -; GFX9-LABEL: v_srem_i128_vv: -; GFX9: ; %bb.0: ; %_udiv-special-cases -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_ashrrev_i32_e32 v20, 31, v3 -; GFX9-NEXT: v_xor_b32_e32 v0, v0, v20 -; GFX9-NEXT: v_xor_b32_e32 v10, v2, v20 -; GFX9-NEXT: v_xor_b32_e32 v1, v1, v20 -; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v20 -; GFX9-NEXT: v_xor_b32_e32 v9, v3, v20 -; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v20, vcc -; GFX9-NEXT: v_ashrrev_i32_e32 v8, 31, v7 -; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v10, v20, vcc -; GFX9-NEXT: v_xor_b32_e32 v4, v4, v8 -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v9, v20, vcc -; GFX9-NEXT: v_xor_b32_e32 v5, v5, v8 -; GFX9-NEXT: v_sub_co_u32_e32 v23, vcc, v4, v8 -; GFX9-NEXT: v_xor_b32_e32 v6, v6, v8 -; GFX9-NEXT: v_subb_co_u32_e32 v21, vcc, v5, v8, vcc -; GFX9-NEXT: v_xor_b32_e32 v7, v7, v8 -; GFX9-NEXT: v_subb_co_u32_e32 v4, vcc, v6, v8, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v8, vcc -; GFX9-NEXT: v_or_b32_e32 v7, v21, v5 -; GFX9-NEXT: v_or_b32_e32 v6, v23, v4 -; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7] -; GFX9-NEXT: v_or_b32_e32 v7, v3, v1 -; GFX9-NEXT: v_or_b32_e32 v6, v2, v0 -; GFX9-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[6:7] -; GFX9-NEXT: v_ffbh_u32_e32 v6, v4 -; GFX9-NEXT: v_add_u32_e32 v6, 32, v6 -; GFX9-NEXT: v_ffbh_u32_e32 v7, v5 -; GFX9-NEXT: v_min_u32_e32 v6, v6, v7 -; GFX9-NEXT: v_ffbh_u32_e32 v7, v23 -; GFX9-NEXT: v_add_u32_e32 v7, 32, v7 -; GFX9-NEXT: v_ffbh_u32_e32 v8, v21 -; GFX9-NEXT: v_min_u32_e32 v7, v7, v8 -; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, 64, v7 -; GFX9-NEXT: v_addc_co_u32_e64 v8, s[6:7], 0, 0, vcc -; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] -; GFX9-NEXT: v_ffbh_u32_e32 v9, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX9-NEXT: v_ffbh_u32_e32 v7, v0 -; GFX9-NEXT: v_add_u32_e32 v7, 32, v7 -; GFX9-NEXT: v_min_u32_e32 v7, v7, v9 -; GFX9-NEXT: v_ffbh_u32_e32 v9, v2 -; GFX9-NEXT: v_add_u32_e32 v9, 32, v9 -; GFX9-NEXT: v_ffbh_u32_e32 v10, v3 -; GFX9-NEXT: v_min_u32_e32 v9, v9, v10 -; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, 0, vcc -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, 64, v9 -; GFX9-NEXT: v_addc_co_u32_e64 v10, s[6:7], 0, 0, vcc -; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] -; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f -; GFX9-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc -; GFX9-NEXT: v_sub_co_u32_e32 v6, vcc, v6, v7 -; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v8, v10, vcc -; GFX9-NEXT: v_mov_b32_e32 v9, 0 -; GFX9-NEXT: v_subbrev_co_u32_e32 v8, vcc, 0, v9, vcc -; GFX9-NEXT: v_subbrev_co_u32_e32 v9, vcc, 0, v9, vcc -; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7] -; GFX9-NEXT: v_or_b32_e32 v13, v7, v9 -; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc -; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9] -; GFX9-NEXT: v_mov_b32_e32 v22, v20 -; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc -; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v10, v11, v10, vcc -; GFX9-NEXT: v_and_b32_e32 v10, 1, v10 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10 -; GFX9-NEXT: v_xor_b32_e32 v10, 0x7f, v6 -; GFX9-NEXT: v_or_b32_e32 v12, v10, v8 -; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], vcc -; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13] -; GFX9-NEXT: s_xor_b64 s[6:7], s[4:5], -1 -; GFX9-NEXT: v_cndmask_b32_e64 v11, v1, 0, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v12, v0, 0, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v10, v3, 0, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v13, v2, 0, s[4:5] -; GFX9-NEXT: s_and_b64 s[4:5], s[6:7], vcc -; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] -; GFX9-NEXT: s_cbranch_execz .LBB2_6 -; GFX9-NEXT: ; %bb.1: ; %udiv-bb1 -; GFX9-NEXT: v_add_co_u32_e32 v24, vcc, 1, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v7, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v26, vcc, 0, v8, vcc -; GFX9-NEXT: v_sub_u32_e32 v13, 0x7f, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, 0, v9, vcc -; GFX9-NEXT: v_sub_u32_e32 v11, 64, v13 -; GFX9-NEXT: v_or_b32_e32 v8, v25, v27 -; GFX9-NEXT: v_or_b32_e32 v7, v24, v26 -; GFX9-NEXT: v_lshlrev_b64 v[9:10], v13, v[0:1] -; GFX9-NEXT: v_lshrrev_b64 v[11:12], v11, v[2:3] -; GFX9-NEXT: v_sub_u32_e32 v6, 63, v6 -; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[7:8] -; GFX9-NEXT: v_lshlrev_b64 v[6:7], v6, v[2:3] -; GFX9-NEXT: v_or_b32_e32 v8, v10, v12 -; GFX9-NEXT: v_or_b32_e32 v9, v9, v11 -; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v13 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v13 -; GFX9-NEXT: v_lshlrev_b64 v[12:13], v13, v[2:3] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[4:5] -; GFX9-NEXT: v_mov_b32_e32 v8, 0 -; GFX9-NEXT: v_mov_b32_e32 v10, 0 -; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v1, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v0, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v13, 0, v13, s[4:5] -; GFX9-NEXT: v_mov_b32_e32 v9, 0 -; GFX9-NEXT: v_mov_b32_e32 v11, 0 -; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, v12, s[4:5] -; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX9-NEXT: s_xor_b64 s[6:7], exec, s[4:5] -; GFX9-NEXT: s_cbranch_execz .LBB2_5 -; GFX9-NEXT: ; %bb.2: ; %udiv-preheader -; GFX9-NEXT: v_sub_u32_e32 v10, 64, v24 -; GFX9-NEXT: v_lshrrev_b64 v[8:9], v24, v[2:3] -; GFX9-NEXT: v_lshlrev_b64 v[10:11], v10, v[0:1] -; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v24 -; GFX9-NEXT: v_or_b32_e32 v10, v8, v10 -; GFX9-NEXT: v_subrev_u32_e32 v8, 64, v24 -; GFX9-NEXT: v_or_b32_e32 v11, v9, v11 -; GFX9-NEXT: v_lshrrev_b64 v[8:9], v8, v[0:1] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v24 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v15, v9, v3, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v10, v8, v10, vcc -; GFX9-NEXT: v_lshrrev_b64 v[8:9], v24, v[0:1] -; GFX9-NEXT: v_cndmask_b32_e64 v14, v10, v2, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v17, 0, v9, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v16, 0, v8, vcc -; GFX9-NEXT: v_add_co_u32_e32 v28, vcc, -1, v23 -; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v21, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v30, vcc, -1, v4, vcc -; GFX9-NEXT: v_mov_b32_e32 v18, 0 -; GFX9-NEXT: v_mov_b32_e32 v10, 0 -; GFX9-NEXT: v_addc_co_u32_e32 v31, vcc, -1, v5, vcc -; GFX9-NEXT: s_mov_b64 s[4:5], 0 -; GFX9-NEXT: v_mov_b32_e32 v19, 0 -; GFX9-NEXT: v_mov_b32_e32 v11, 0 -; GFX9-NEXT: v_mov_b32_e32 v9, 0 -; GFX9-NEXT: .LBB2_3: ; %udiv-do-while -; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX9-NEXT: v_lshrrev_b32_e32 v32, 31, v15 -; GFX9-NEXT: v_lshlrev_b64 v[14:15], 1, v[14:15] -; GFX9-NEXT: v_lshrrev_b32_e32 v33, 31, v7 -; GFX9-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7] -; GFX9-NEXT: v_lshrrev_b32_e32 v8, 31, v13 -; GFX9-NEXT: v_lshlrev_b64 v[16:17], 1, v[16:17] -; GFX9-NEXT: v_or_b32_e32 v14, v14, v33 -; GFX9-NEXT: v_or3_b32 v6, v6, v8, v10 -; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v28, v14 -; GFX9-NEXT: v_or_b32_e32 v16, v16, v32 -; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, v29, v15, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, v30, v16, vcc -; GFX9-NEXT: v_lshlrev_b64 v[12:13], 1, v[12:13] -; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, v31, v17, vcc -; GFX9-NEXT: v_ashrrev_i32_e32 v8, 31, v8 -; GFX9-NEXT: v_or_b32_e32 v12, v18, v12 -; GFX9-NEXT: v_and_b32_e32 v18, v8, v23 -; GFX9-NEXT: v_or_b32_e32 v13, v19, v13 -; GFX9-NEXT: v_and_b32_e32 v19, v8, v21 -; GFX9-NEXT: v_sub_co_u32_e32 v14, vcc, v14, v18 -; GFX9-NEXT: v_and_b32_e32 v32, v8, v4 -; GFX9-NEXT: v_subb_co_u32_e32 v15, vcc, v15, v19, vcc -; GFX9-NEXT: v_and_b32_e32 v33, v8, v5 -; GFX9-NEXT: v_subb_co_u32_e32 v16, vcc, v16, v32, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v17, vcc, v17, v33, vcc -; GFX9-NEXT: v_add_co_u32_e32 v24, vcc, -1, v24 -; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v25, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v26, vcc, -1, v26, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, -1, v27, vcc -; GFX9-NEXT: v_or_b32_e32 v18, v24, v26 -; GFX9-NEXT: v_or_b32_e32 v19, v25, v27 -; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[18:19] -; GFX9-NEXT: v_and_b32_e32 v8, 1, v8 -; GFX9-NEXT: v_mov_b32_e32 v19, v9 -; GFX9-NEXT: v_or3_b32 v7, v7, 0, v11 -; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX9-NEXT: v_mov_b32_e32 v18, v8 -; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX9-NEXT: s_cbranch_execnz .LBB2_3 -; GFX9-NEXT: ; %bb.4: ; %Flow -; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX9-NEXT: .LBB2_5: ; %Flow2 -; GFX9-NEXT: s_or_b64 exec, exec, s[6:7] -; GFX9-NEXT: v_lshlrev_b64 v[14:15], 1, v[12:13] -; GFX9-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7] -; GFX9-NEXT: v_lshrrev_b32_e32 v12, 31, v13 -; GFX9-NEXT: v_or3_b32 v11, v7, 0, v11 -; GFX9-NEXT: v_or3_b32 v12, v6, v12, v10 -; GFX9-NEXT: v_or_b32_e32 v10, v9, v15 -; GFX9-NEXT: v_or_b32_e32 v13, v8, v14 -; GFX9-NEXT: .LBB2_6: ; %Flow3 -; GFX9-NEXT: s_or_b64 exec, exec, s[8:9] -; GFX9-NEXT: v_mul_lo_u32 v16, v13, v5 -; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v23, v13, 0 -; GFX9-NEXT: v_mov_b32_e32 v15, 0 -; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v13, v4, 0 -; GFX9-NEXT: v_mov_b32_e32 v14, v6 -; GFX9-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v21, v13, v[14:15] -; GFX9-NEXT: v_mul_lo_u32 v9, v10, v4 -; GFX9-NEXT: v_mul_lo_u32 v11, v11, v23 -; GFX9-NEXT: v_mov_b32_e32 v4, v14 -; GFX9-NEXT: v_mov_b32_e32 v14, v15 -; GFX9-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v23, v10, v[13:14] -; GFX9-NEXT: v_add3_u32 v8, v8, v16, v9 -; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v12, v23, v[7:8] -; GFX9-NEXT: v_mov_b32_e32 v8, v14 -; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v4, v8 -; GFX9-NEXT: v_addc_co_u32_e64 v9, s[4:5], 0, 0, vcc -; GFX9-NEXT: v_mul_lo_u32 v12, v12, v21 -; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v21, v10, v[8:9] -; GFX9-NEXT: v_add3_u32 v4, v11, v7, v12 -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v8, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v9, v4, vcc -; GFX9-NEXT: v_mov_b32_e32 v7, v13 -; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v5 -; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v7, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v6, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v4, vcc -; GFX9-NEXT: v_xor_b32_e32 v5, v0, v20 -; GFX9-NEXT: v_xor_b32_e32 v0, v2, v20 -; GFX9-NEXT: v_xor_b32_e32 v4, v1, v22 -; GFX9-NEXT: v_xor_b32_e32 v1, v3, v22 -; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v20 -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v22, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v5, v20, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v22, vcc -; GFX9-NEXT: s_setpc_b64 s[30:31] -; -; GFX9-O0-LABEL: v_srem_i128_vv: -; GFX9-O0: ; %bb.0: ; %_udiv-special-cases -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] -; GFX9-O0-NEXT: ; implicit-def: $vgpr8 : SGPR spill to VGPR lane -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v6 -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v2 -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v1 -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v0 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec -; GFX9-O0-NEXT: s_waitcnt vmcnt(1) -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v1 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 -; GFX9-O0-NEXT: s_mov_b32 s4, 63 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v14 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v15 -; GFX9-O0-NEXT: v_ashrrev_i64 v[12:13], s4, v[6:7] -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v13 -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v8 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v9 -; GFX9-O0-NEXT: v_ashrrev_i64 v[6:7], s4, v[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v15 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v13 -; GFX9-O0-NEXT: v_xor_b32_e64 v1, v1, v10 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v14 -; GFX9-O0-NEXT: v_xor_b32_e64 v13, v11, v12 -; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v5 -; GFX9-O0-NEXT: v_xor_b32_e64 v1, v1, v10 -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec -; GFX9-O0-NEXT: v_xor_b32_e64 v15, v4, v12 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v7 -; GFX9-O0-NEXT: v_xor_b32_e64 v1, v1, v4 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v8 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_xor_b32_e64 v7, v5, v6 -; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3 -; GFX9-O0-NEXT: v_xor_b32_e64 v1, v1, v4 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_xor_b32_e64 v2, v2, v6 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v15 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v16 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v14 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v9, vcc, v9, v12 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v10, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v13, vcc, v11, v12, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v5, vcc, v5, v10, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v2 -; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v8 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v6 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v4, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v11, vcc, v5, v6, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v4, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v2 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v12 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v2 -; GFX9-O0-NEXT: v_or_b32_e64 v3, v8, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 -; GFX9-O0-NEXT: v_or_b32_e64 v1, v5, v6 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 -; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 0 -; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 1 -; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[1:2], s[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v14 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 -; GFX9-O0-NEXT: v_or_b32_e64 v15, v4, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 -; GFX9-O0-NEXT: v_or_b32_e64 v9, v3, v1 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v15 -; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[9:10], s[6:7] -; GFX9-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9] -; GFX9-O0-NEXT: v_ffbh_u32_e64 v6, v6 -; GFX9-O0-NEXT: s_mov_b32 s9, 32 -; GFX9-O0-NEXT: v_add_u32_e64 v6, v6, s9 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v7, v7 -; GFX9-O0-NEXT: v_min_u32_e64 v6, v6, v7 -; GFX9-O0-NEXT: s_mov_b32 s8, 0 -; GFX9-O0-NEXT: ; implicit-def: $sgpr10 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, s8 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v7 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v5 -; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v8, v8 -; GFX9-O0-NEXT: v_min_u32_e64 v15, v5, v8 -; GFX9-O0-NEXT: ; implicit-def: $sgpr10 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v5 -; GFX9-O0-NEXT: s_mov_b64 s[10:11], 64 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v15 -; GFX9-O0-NEXT: s_mov_b32 s12, s10 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16 -; GFX9-O0-NEXT: s_mov_b32 s14, s11 -; GFX9-O0-NEXT: v_add_co_u32_e64 v8, s[12:13], v8, s12 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, s14 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[12:13], v5, v9, s[12:13] -; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 -; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[12:13], v[11:12], s[6:7] -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v10, s[12:13] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v8 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v9, v6, v7, s[12:13] -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v1 -; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v6, v2 -; GFX9-O0-NEXT: v_min_u32_e64 v6, v5, v6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v7 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v3 -; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v11, v4 -; GFX9-O0-NEXT: v_min_u32_e64 v15, v5, v11 -; GFX9-O0-NEXT: ; implicit-def: $sgpr9 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v15 -; GFX9-O0-NEXT: s_mov_b32 s8, s10 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16 -; GFX9-O0-NEXT: s_mov_b32 s10, s11 -; GFX9-O0-NEXT: v_add_co_u32_e64 v11, s[8:9], v11, s8 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, s10 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[8:9], v5, v12, s[8:9] -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 -; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[8:9], v[13:14], s[6:7] -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[8:9] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[8:9] -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 -; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 -; GFX9-O0-NEXT: s_mov_b32 s10, s6 -; GFX9-O0-NEXT: s_mov_b32 s11, s7 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v5, vcc, v5, v8 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v9, vcc, v6, v7, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v7, s10 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, s10 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v8, vcc, v6, v7, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v7, s11 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, s11 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v6, v7, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v9 -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v8 -; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 -; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] -; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[8:9], s[8:9] -; GFX9-O0-NEXT: s_mov_b64 s[12:13], 0x7f -; GFX9-O0-NEXT: s_mov_b64 s[14:15], s[12:13] -; GFX9-O0-NEXT: v_cmp_gt_u64_e64 s[14:15], v[5:6], s[14:15] -; GFX9-O0-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[14:15] -; GFX9-O0-NEXT: s_mov_b64 s[14:15], s[6:7] -; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[14:15], v[8:9], s[14:15] -; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[14:15] -; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, v7, v10, s[8:9] -; GFX9-O0-NEXT: v_and_b32_e64 v7, 1, v7 -; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[8:9], v7, 1 -; GFX9-O0-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9] -; GFX9-O0-NEXT: s_mov_b64 s[4:5], -1 -; GFX9-O0-NEXT: s_xor_b64 s[4:5], s[8:9], s[4:5] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 -; GFX9-O0-NEXT: s_mov_b32 s14, s13 -; GFX9-O0-NEXT: v_xor_b32_e64 v7, v7, s14 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13 -; GFX9-O0-NEXT: v_xor_b32_e64 v5, v5, s12 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v9 -; GFX9-O0-NEXT: v_or_b32_e64 v7, v7, v10 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v8 -; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v6 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7 -; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[5:6], s[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v2, v5, s[8:9] -; GFX9-O0-NEXT: v_mov_b32_e32 v2, s10 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v1, v1, v2, s[8:9] -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v4, v5, s[8:9] -; GFX9-O0-NEXT: v_mov_b32_e32 v4, s10 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[8:9] -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v5 -; GFX9-O0-NEXT: s_and_b64 s[6:7], s[4:5], s[6:7] -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 s[4:5], exec -; GFX9-O0-NEXT: v_writelane_b32 v0, s4, 2 -; GFX9-O0-NEXT: v_writelane_b32 v0, s5, 3 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7] -; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] -; GFX9-O0-NEXT: s_cbranch_execz .LBB2_3 -; GFX9-O0-NEXT: s_branch .LBB2_8 -; GFX9-O0-NEXT: .LBB2_1: ; %Flow -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s4, v0, 4 -; GFX9-O0-NEXT: v_readlane_b32 s5, v0, 5 -; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX9-O0-NEXT: ; %bb.2: ; %Flow -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_waitcnt vmcnt(6) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_branch .LBB2_5 -; GFX9-O0-NEXT: .LBB2_3: ; %Flow2 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s4, v4, 2 -; GFX9-O0-NEXT: v_readlane_b32 s5, v4, 3 -; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_branch .LBB2_9 -; GFX9-O0-NEXT: .LBB2_4: ; %udiv-loop-exit -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b32 s4, 1 -; GFX9-O0-NEXT: s_waitcnt vmcnt(2) -; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[0:1] -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_lshlrev_b64 v[9:10], s4, v[9:10] -; GFX9-O0-NEXT: s_mov_b32 s4, 63 -; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1] -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v8 -; GFX9-O0-NEXT: v_or3_b32 v4, v4, v11, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v9 -; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v7 -; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v6 -; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5 -; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v4 -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_branch .LBB2_3 -; GFX9-O0-NEXT: .LBB2_5: ; %Flow1 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s4, v8, 6 -; GFX9-O0-NEXT: v_readlane_b32 s5, v8, 7 -; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_branch .LBB2_4 -; GFX9-O0-NEXT: .LBB2_6: ; %udiv-do-while -; GFX9-O0-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s6, v16, 8 -; GFX9-O0-NEXT: v_readlane_b32 s7, v16, 9 -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b32 s4, 63 -; GFX9-O0-NEXT: s_waitcnt vmcnt(16) -; GFX9-O0-NEXT: v_lshrrev_b64 v[29:30], s4, v[2:3] -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v30 -; GFX9-O0-NEXT: s_mov_b32 s5, 1 -; GFX9-O0-NEXT: v_lshlrev_b64 v[23:24], s5, v[23:24] -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v24 -; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v29 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v23 -; GFX9-O0-NEXT: v_or_b32_e64 v23, v5, v10 -; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v24, v4 -; GFX9-O0-NEXT: v_lshlrev_b64 v[29:30], s5, v[2:3] -; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], s4, v[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v30 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 -; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v29 -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec -; GFX9-O0-NEXT: v_or_b32_e64 v4, v3, v4 -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v2 -; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s5, v[0:1] -; GFX9-O0-NEXT: v_lshlrev_b64 v[29:30], s5, v[6:7] -; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v30 -; GFX9-O0-NEXT: s_waitcnt vmcnt(10) -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v28 -; GFX9-O0-NEXT: v_or3_b32 v6, v6, v7, v10 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v29 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v27 -; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v7 -; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3 -; GFX9-O0-NEXT: s_waitcnt vmcnt(8) -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v26 -; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v25 -; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v4 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v23 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v24 -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_mov_b32_e32 v13, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v14 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v15 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v13, vcc, v13, v6 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v12, vcc, v12, v10, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v11, vcc, v11, v4, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v7, v5, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v7 -; GFX9-O0-NEXT: v_ashrrev_i64 v[13:14], s4, v[11:12] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v14 -; GFX9-O0-NEXT: s_mov_b64 s[4:5], 1 -; GFX9-O0-NEXT: s_mov_b32 s8, s5 -; GFX9-O0-NEXT: v_and_b32_e64 v12, v7, s8 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v13 -; GFX9-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5 -; GFX9-O0-NEXT: v_and_b32_e64 v14, v11, s4 -; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v13, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v23, v22 -; GFX9-O0-NEXT: v_and_b32_e64 v23, v7, v23 -; GFX9-O0-NEXT: v_and_b32_e64 v21, v11, v21 -; GFX9-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v22, v23 -; GFX9-O0-NEXT: v_mov_b32_e32 v23, v20 -; GFX9-O0-NEXT: v_and_b32_e64 v7, v7, v23 -; GFX9-O0-NEXT: v_and_b32_e64 v23, v11, v19 -; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v24, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v23 -; GFX9-O0-NEXT: v_mov_b32_e32 v19, v24 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v21 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v22 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v6, vcc, v6, v20 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v10, vcc, v10, v19, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v4, vcc, v4, v11, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v5, v7, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v10 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v8 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 killed $vgpr8_vgpr9 killed $exec -; GFX9-O0-NEXT: s_mov_b64 s[8:9], -1 -; GFX9-O0-NEXT: s_mov_b32 s5, s8 -; GFX9-O0-NEXT: s_mov_b32 s4, s9 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v18 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, s5 -; GFX9-O0-NEXT: v_add_co_u32_e32 v20, vcc, v11, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, s4 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v9, vcc, v9, v11, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v11, s5 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v17, vcc, v10, v11, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v10, s4 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v8, vcc, v8, v10, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr20 killed $vgpr20 def $vgpr20_vgpr21 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v21, v9 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v8 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v18 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v20 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v21 -; GFX9-O0-NEXT: v_mov_b32_e32 v22, v18 -; GFX9-O0-NEXT: v_mov_b32_e32 v19, v21 -; GFX9-O0-NEXT: v_or_b32_e64 v19, v19, v22 -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, v20 -; GFX9-O0-NEXT: v_or_b32_e64 v17, v17, v18 -; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v19 -; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[17:18], v[12:13] -; GFX9-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, v2 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, v0 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v15 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, v14 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, v12 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5] -; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 4 -; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 5 -; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5] -; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 8 -; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 9 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX9-O0-NEXT: s_cbranch_execnz .LBB2_6 -; GFX9-O0-NEXT: s_branch .LBB2_1 -; GFX9-O0-NEXT: .LBB2_7: ; %udiv-preheader -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_waitcnt vmcnt(9) -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], v4, v[21:22] -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7 -; GFX9-O0-NEXT: s_mov_b32 s6, 64 -; GFX9-O0-NEXT: v_sub_u32_e64 v12, s6, v4 -; GFX9-O0-NEXT: v_lshlrev_b64 v[23:24], v12, v[19:20] -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v24 -; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v12 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v23 -; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v7 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v7 -; GFX9-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v4, s6 -; GFX9-O0-NEXT: v_sub_u32_e64 v5, v4, s6 -; GFX9-O0-NEXT: v_lshrrev_b64 v[23:24], v5, v[19:20] -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v24 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v12, s[4:5] -; GFX9-O0-NEXT: s_mov_b32 s6, 0 -; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v4, s6 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v22 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v12, s[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v23 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v21 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[6:7] -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[19:20] -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v5 -; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 -; GFX9-O0-NEXT: s_mov_b32 s8, s7 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, s8 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v12, v12, v15, s[4:5] -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4 -; GFX9-O0-NEXT: s_mov_b32 s8, s6 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[4:5] -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v14 -; GFX9-O0-NEXT: s_mov_b64 s[8:9], -1 -; GFX9-O0-NEXT: s_mov_b32 s5, s8 -; GFX9-O0-NEXT: s_mov_b32 s4, s9 -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v13, v18 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, s5 -; GFX9-O0-NEXT: v_add_co_u32_e32 v12, vcc, v12, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, s4 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v17, vcc, v15, v17, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v15, s5 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v14, vcc, v14, v15, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v15, s4 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v13, vcc, v13, v15, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v13 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v13, v17 -; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] -; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v15, s9 -; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8 -; GFX9-O0-NEXT: v_mov_b32_e32 v13, s7 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, s6 -; GFX9-O0-NEXT: v_writelane_b32 v16, s4, 8 -; GFX9-O0-NEXT: v_writelane_b32 v16, s5, 9 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_branch .LBB2_6 -; GFX9-O0-NEXT: .LBB2_8: ; %udiv-bb1 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 s[6:7], 1 -; GFX9-O0-NEXT: s_mov_b32 s5, s6 -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v1 -; GFX9-O0-NEXT: s_mov_b32 s4, s7 -; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 -; GFX9-O0-NEXT: s_mov_b32 s8, s6 -; GFX9-O0-NEXT: s_mov_b32 s9, s7 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s5 -; GFX9-O0-NEXT: v_add_co_u32_e32 v9, vcc, v4, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s4 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v5, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s9 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v5, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v2 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b32 s4, 0x7f -; GFX9-O0-NEXT: v_sub_u32_e64 v3, s4, v4 -; GFX9-O0-NEXT: v_lshlrev_b64 v[5:6], v3, v[11:12] -; GFX9-O0-NEXT: v_mov_b32_e32 v13, v6 -; GFX9-O0-NEXT: s_mov_b32 s4, 64 -; GFX9-O0-NEXT: v_sub_u32_e64 v14, s4, v3 -; GFX9-O0-NEXT: v_lshrrev_b64 v[14:15], v14, v[7:8] -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v15 -; GFX9-O0-NEXT: v_or_b32_e64 v13, v13, v16 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v14 -; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v6 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v6 -; GFX9-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v3, s4 -; GFX9-O0-NEXT: s_mov_b32 s10, 63 -; GFX9-O0-NEXT: v_sub_u32_e64 v4, s10, v4 -; GFX9-O0-NEXT: v_lshlrev_b64 v[13:14], v4, v[7:8] -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v15, s[4:5] -; GFX9-O0-NEXT: s_mov_b32 s10, 0 -; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[10:11], v3, s10 -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v15, s[10:11] -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v13 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[4:5] -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[10:11] -; GFX9-O0-NEXT: ; implicit-def: $sgpr10 -; GFX9-O0-NEXT: ; implicit-def: $sgpr10 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v4 -; GFX9-O0-NEXT: v_lshlrev_b64 v[7:8], v3, v[7:8] -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v8 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, s9 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[4:5] -; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr7_vgpr8 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, v4, v7, s[4:5] -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v3 -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v10 -; GFX9-O0-NEXT: v_or_b32_e64 v3, v3, v4 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v9 -; GFX9-O0-NEXT: v_or_b32_e64 v1, v1, v2 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[4:5], v[1:2], s[6:7] -; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v3, s8 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, s9 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, s6 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, s7 -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 s[6:7], exec -; GFX9-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5] -; GFX9-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7] -; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 6 -; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 7 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] -; GFX9-O0-NEXT: s_cbranch_execz .LBB2_5 -; GFX9-O0-NEXT: s_branch .LBB2_7 -; GFX9-O0-NEXT: .LBB2_9: ; %udiv-end -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b32 s4, 32 -; GFX9-O0-NEXT: s_waitcnt vmcnt(2) -; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[5:6] -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v17 -; GFX9-O0-NEXT: v_mul_lo_u32 v3, v1, v0 -; GFX9-O0-NEXT: v_lshrrev_b64 v[17:18], s4, v[17:18] -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v17 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mul_lo_u32 v2, v5, v2 -; GFX9-O0-NEXT: v_mad_u64_u32 v[17:18], s[6:7], v5, v0, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v18 -; GFX9-O0-NEXT: v_add3_u32 v2, v0, v2, v3 -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v0 -; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[2:3] -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v3 -; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 killed $vgpr17_vgpr18 killed $exec -; GFX9-O0-NEXT: s_mov_b32 s5, 0 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v18 -; GFX9-O0-NEXT: v_or_b32_e64 v0, v0, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v17 -; GFX9-O0-NEXT: v_or_b32_e64 v17, v2, v3 -; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v0 -; GFX9-O0-NEXT: v_lshrrev_b64 v[2:3], s4, v[19:20] -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v11 -; GFX9-O0-NEXT: v_mul_lo_u32 v3, v2, v6 -; GFX9-O0-NEXT: v_lshrrev_b64 v[11:12], s4, v[11:12] -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 killed $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v19 -; GFX9-O0-NEXT: v_mul_lo_u32 v11, v11, v0 -; GFX9-O0-NEXT: v_mad_u64_u32 v[19:20], s[6:7], v2, v0, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v20 -; GFX9-O0-NEXT: v_add3_u32 v2, v2, v3, v11 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, s6 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[2:3] -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v3 -; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 killed $vgpr19_vgpr20 killed $exec -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v20 -; GFX9-O0-NEXT: v_or_b32_e64 v11, v11, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v19 -; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v18 -; GFX9-O0-NEXT: v_add_co_u32_e64 v17, s[6:7], v11, v12 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v2, s[6:7], v2, v3, s[6:7] -; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v2 -; GFX9-O0-NEXT: v_mad_u64_u32 v[19:20], s[6:7], v6, v1, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v19 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v19, v20 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, s6 -; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v12 -; GFX9-O0-NEXT: v_lshlrev_b64 v[19:20], s4, v[19:20] -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v20 -; GFX9-O0-NEXT: v_or_b32_e64 v11, v11, v12 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v19 -; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_mad_u64_u32 v[19:20], s[6:7], v6, v5, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v19 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v19, v20 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: v_mov_b32_e32 v21, s6 -; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v21 -; GFX9-O0-NEXT: v_lshlrev_b64 v[19:20], s4, v[19:20] -; GFX9-O0-NEXT: v_mov_b32_e32 v21, v20 -; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v21 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 killed $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v19 -; GFX9-O0-NEXT: v_or_b32_e64 v23, v11, v12 -; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v24, v6 -; GFX9-O0-NEXT: v_mad_u64_u32 v[11:12], s[6:7], v0, v5, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v21, v12 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v22, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v23 -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v21 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v24 -; GFX9-O0-NEXT: v_mov_b32_e32 v19, v22 -; GFX9-O0-NEXT: v_add_co_u32_e64 v5, s[6:7], v5, v20 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v19, s[6:7], v6, v19, s[6:7] -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v19 -; GFX9-O0-NEXT: v_mov_b32_e32 v19, v6 -; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0xffffffff -; GFX9-O0-NEXT: s_mov_b32 s8, s7 -; GFX9-O0-NEXT: v_and_b32_e64 v19, v19, s8 -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v5 -; GFX9-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 killed $sgpr6_sgpr7 -; GFX9-O0-NEXT: v_and_b32_e64 v21, v20, s6 -; GFX9-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v22, v19 -; GFX9-O0-NEXT: v_mad_u64_u32 v[19:20], s[6:7], v0, v1, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v23, v19 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v24, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v24 -; GFX9-O0-NEXT: v_mov_b32_e32 v19, v20 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, s6 -; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v1 -; GFX9-O0-NEXT: v_lshlrev_b64 v[19:20], s4, v[19:20] -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v20 -; GFX9-O0-NEXT: v_or_b32_e64 v0, v0, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v23 -; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 killed $vgpr19_vgpr20 killed $exec -; GFX9-O0-NEXT: v_or_b32_e64 v23, v1, v19 -; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v24, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v23 -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v21 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v24 -; GFX9-O0-NEXT: v_mov_b32_e32 v19, v22 -; GFX9-O0-NEXT: v_add_co_u32_e64 v0, s[6:7], v0, v20 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v19, s[6:7], v1, v19, s[6:7] -; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v19 -; GFX9-O0-NEXT: v_lshrrev_b64 v[21:22], s4, v[0:1] -; GFX9-O0-NEXT: v_lshrrev_b64 v[5:6], s4, v[5:6] -; GFX9-O0-NEXT: v_mov_b32_e32 v19, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v21 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v22 -; GFX9-O0-NEXT: v_add_co_u32_e64 v19, s[6:7], v19, v20 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v5, v6, s[6:7] -; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v19 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v20 -; GFX9-O0-NEXT: v_add_co_u32_e64 v19, s[6:7], v5, v6 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v2, s[6:7], v2, v3, s[6:7] -; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v19 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v20 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v18 -; GFX9-O0-NEXT: v_add_co_u32_e64 v2, s[6:7], v2, v6 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v3, v5, s[6:7] -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 -; GFX9-O0-NEXT: v_lshlrev_b64 v[0:1], s4, v[0:1] -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v1 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 killed $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 -; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v11 -; GFX9-O0-NEXT: v_or_b32_e64 v0, v0, v1 -; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v15 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v16 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v14 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v12 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v11, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v5, vcc, v5, v6, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v2, vcc, v1, v2, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3 -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v10 -; GFX9-O0-NEXT: v_xor_b32_e64 v3, v3, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 -; GFX9-O0-NEXT: v_xor_b32_e64 v9, v6, v5 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v8 -; GFX9-O0-NEXT: v_xor_b32_e64 v3, v3, v6 -; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v7 -; GFX9-O0-NEXT: v_xor_b32_e64 v0, v0, v8 -; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v10 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v7, vcc, v7, v8 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v6, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v5, vcc, v3, v5, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v2, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v1 -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5 -; GFX9-O0-NEXT: v_lshrrev_b64 v[7:8], s4, v[7:8] -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 -; GFX9-O0-NEXT: v_lshrrev_b64 v[5:6], s4, v[5:6] -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 -; GFX9-O0-NEXT: ; kill: killed $vgpr4 -; GFX9-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_nop 0 -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: s_setpc_b64 s[30:31] - %div = srem i128 %lhs, %rhs - ret i128 %div -} - -define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) { -; GFX9-LABEL: v_urem_i128_vv: -; GFX9: ; %bb.0: ; %_udiv-special-cases -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_or_b32_e32 v9, v5, v7 -; GFX9-NEXT: v_or_b32_e32 v8, v4, v6 -; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] -; GFX9-NEXT: v_or_b32_e32 v9, v1, v3 -; GFX9-NEXT: v_or_b32_e32 v8, v0, v2 -; GFX9-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[8:9] -; GFX9-NEXT: v_ffbh_u32_e32 v8, v6 -; GFX9-NEXT: v_add_u32_e32 v8, 32, v8 -; GFX9-NEXT: v_ffbh_u32_e32 v9, v7 -; GFX9-NEXT: v_min_u32_e32 v8, v8, v9 -; GFX9-NEXT: v_ffbh_u32_e32 v9, v4 -; GFX9-NEXT: v_add_u32_e32 v9, 32, v9 -; GFX9-NEXT: v_ffbh_u32_e32 v10, v5 -; GFX9-NEXT: v_min_u32_e32 v9, v9, v10 -; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, 64, v9 -; GFX9-NEXT: v_addc_co_u32_e64 v10, s[6:7], 0, 0, vcc -; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7] -; GFX9-NEXT: v_ffbh_u32_e32 v11, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc -; GFX9-NEXT: v_ffbh_u32_e32 v9, v2 -; GFX9-NEXT: v_add_u32_e32 v9, 32, v9 -; GFX9-NEXT: v_min_u32_e32 v9, v9, v11 -; GFX9-NEXT: v_ffbh_u32_e32 v11, v0 -; GFX9-NEXT: v_add_u32_e32 v11, 32, v11 -; GFX9-NEXT: v_ffbh_u32_e32 v12, v1 -; GFX9-NEXT: v_min_u32_e32 v11, v11, v12 -; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc -; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, 64, v11 -; GFX9-NEXT: v_addc_co_u32_e64 v12, s[6:7], 0, 0, vcc -; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] -; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f -; GFX9-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, 0, vcc -; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v8, v9 -; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, v10, v12, vcc -; GFX9-NEXT: v_mov_b32_e32 v11, 0 -; GFX9-NEXT: v_subbrev_co_u32_e32 v10, vcc, 0, v11, vcc -; GFX9-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v11, vcc -; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc -; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc -; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v12, v13, v12, vcc -; GFX9-NEXT: v_and_b32_e32 v12, 1, v12 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v12 -; GFX9-NEXT: v_xor_b32_e32 v12, 0x7f, v8 -; GFX9-NEXT: v_or_b32_e32 v13, v9, v11 -; GFX9-NEXT: v_or_b32_e32 v12, v12, v10 -; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], vcc -; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13] -; GFX9-NEXT: s_xor_b64 s[6:7], s[4:5], -1 -; GFX9-NEXT: v_cndmask_b32_e64 v15, v3, 0, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v14, v2, 0, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v13, v1, 0, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v12, v0, 0, s[4:5] -; GFX9-NEXT: s_and_b64 s[4:5], s[6:7], vcc -; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] -; GFX9-NEXT: s_cbranch_execz .LBB3_6 -; GFX9-NEXT: ; %bb.1: ; %udiv-bb1 -; GFX9-NEXT: v_add_co_u32_e32 v22, vcc, 1, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v23, vcc, 0, v9, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, 0, v10, vcc -; GFX9-NEXT: v_sub_u32_e32 v15, 0x7f, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v11, vcc -; GFX9-NEXT: v_sub_u32_e32 v13, 64, v15 -; GFX9-NEXT: v_or_b32_e32 v10, v23, v25 -; GFX9-NEXT: v_or_b32_e32 v9, v22, v24 -; GFX9-NEXT: v_lshlrev_b64 v[11:12], v15, v[2:3] -; GFX9-NEXT: v_lshrrev_b64 v[13:14], v13, v[0:1] -; GFX9-NEXT: v_sub_u32_e32 v8, 63, v8 -; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[9:10] -; GFX9-NEXT: v_lshlrev_b64 v[8:9], v8, v[0:1] -; GFX9-NEXT: v_or_b32_e32 v10, v12, v14 -; GFX9-NEXT: v_or_b32_e32 v11, v11, v13 -; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v15 -; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v10, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v11, s[4:5] -; GFX9-NEXT: v_lshlrev_b64 v[10:11], v15, v[0:1] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v15 -; GFX9-NEXT: v_mov_b32_e32 v12, 0 -; GFX9-NEXT: v_mov_b32_e32 v14, 0 -; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v3, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v2, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, v11, s[4:5] -; GFX9-NEXT: v_mov_b32_e32 v13, 0 -; GFX9-NEXT: v_mov_b32_e32 v15, 0 -; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, v10, s[4:5] -; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX9-NEXT: s_xor_b64 s[6:7], exec, s[4:5] -; GFX9-NEXT: s_cbranch_execz .LBB3_5 -; GFX9-NEXT: ; %bb.2: ; %udiv-preheader -; GFX9-NEXT: v_sub_u32_e32 v14, 64, v22 -; GFX9-NEXT: v_lshrrev_b64 v[12:13], v22, v[0:1] -; GFX9-NEXT: v_lshlrev_b64 v[14:15], v14, v[2:3] -; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v22 -; GFX9-NEXT: v_or_b32_e32 v14, v12, v14 -; GFX9-NEXT: v_subrev_u32_e32 v12, 64, v22 -; GFX9-NEXT: v_or_b32_e32 v15, v13, v15 -; GFX9-NEXT: v_lshrrev_b64 v[12:13], v12, v[2:3] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v22 -; GFX9-NEXT: v_cndmask_b32_e32 v13, v13, v15, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v17, v13, v1, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v14, v12, v14, vcc -; GFX9-NEXT: v_lshrrev_b64 v[12:13], v22, v[2:3] -; GFX9-NEXT: v_cndmask_b32_e64 v16, v14, v0, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v19, 0, v13, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v18, 0, v12, vcc -; GFX9-NEXT: v_add_co_u32_e32 v26, vcc, -1, v4 -; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, -1, v5, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v28, vcc, -1, v6, vcc -; GFX9-NEXT: v_mov_b32_e32 v20, 0 -; GFX9-NEXT: v_mov_b32_e32 v14, 0 -; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v7, vcc -; GFX9-NEXT: s_mov_b64 s[4:5], 0 -; GFX9-NEXT: v_mov_b32_e32 v21, 0 -; GFX9-NEXT: v_mov_b32_e32 v15, 0 -; GFX9-NEXT: v_mov_b32_e32 v13, 0 -; GFX9-NEXT: .LBB3_3: ; %udiv-do-while -; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX9-NEXT: v_lshrrev_b32_e32 v12, 31, v11 -; GFX9-NEXT: v_lshlrev_b64 v[10:11], 1, v[10:11] -; GFX9-NEXT: v_lshlrev_b64 v[18:19], 1, v[18:19] -; GFX9-NEXT: v_or_b32_e32 v10, v20, v10 -; GFX9-NEXT: v_lshrrev_b32_e32 v20, 31, v17 -; GFX9-NEXT: v_lshlrev_b64 v[16:17], 1, v[16:17] -; GFX9-NEXT: v_or_b32_e32 v18, v18, v20 -; GFX9-NEXT: v_lshrrev_b32_e32 v20, 31, v9 -; GFX9-NEXT: v_or_b32_e32 v16, v16, v20 -; GFX9-NEXT: v_sub_co_u32_e32 v20, vcc, v26, v16 -; GFX9-NEXT: v_subb_co_u32_e32 v20, vcc, v27, v17, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v20, vcc, v28, v18, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v20, vcc, v29, v19, vcc -; GFX9-NEXT: v_ashrrev_i32_e32 v30, 31, v20 -; GFX9-NEXT: v_and_b32_e32 v20, v30, v4 -; GFX9-NEXT: v_sub_co_u32_e32 v16, vcc, v16, v20 -; GFX9-NEXT: v_and_b32_e32 v20, v30, v5 -; GFX9-NEXT: v_subb_co_u32_e32 v17, vcc, v17, v20, vcc -; GFX9-NEXT: v_and_b32_e32 v20, v30, v6 -; GFX9-NEXT: v_subb_co_u32_e32 v18, vcc, v18, v20, vcc -; GFX9-NEXT: v_and_b32_e32 v20, v30, v7 -; GFX9-NEXT: v_subb_co_u32_e32 v19, vcc, v19, v20, vcc -; GFX9-NEXT: v_add_co_u32_e32 v22, vcc, -1, v22 -; GFX9-NEXT: v_addc_co_u32_e32 v23, vcc, -1, v23, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, -1, v24, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v25, vcc -; GFX9-NEXT: v_or_b32_e32 v11, v21, v11 -; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9] -; GFX9-NEXT: v_or_b32_e32 v20, v22, v24 -; GFX9-NEXT: v_or_b32_e32 v21, v23, v25 -; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[20:21] -; GFX9-NEXT: v_or3_b32 v8, v8, v12, v14 -; GFX9-NEXT: v_and_b32_e32 v12, 1, v30 -; GFX9-NEXT: v_mov_b32_e32 v21, v13 -; GFX9-NEXT: v_or3_b32 v9, v9, 0, v15 -; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX9-NEXT: v_mov_b32_e32 v20, v12 -; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX9-NEXT: s_cbranch_execnz .LBB3_3 -; GFX9-NEXT: ; %bb.4: ; %Flow -; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX9-NEXT: .LBB3_5: ; %Flow2 -; GFX9-NEXT: s_or_b64 exec, exec, s[6:7] -; GFX9-NEXT: v_lshlrev_b64 v[16:17], 1, v[10:11] -; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9] -; GFX9-NEXT: v_lshrrev_b32_e32 v10, 31, v11 -; GFX9-NEXT: v_or3_b32 v15, v9, 0, v15 -; GFX9-NEXT: v_or3_b32 v14, v8, v10, v14 -; GFX9-NEXT: v_or_b32_e32 v13, v13, v17 -; GFX9-NEXT: v_or_b32_e32 v12, v12, v16 -; GFX9-NEXT: .LBB3_6: ; %Flow3 -; GFX9-NEXT: s_or_b64 exec, exec, s[8:9] -; GFX9-NEXT: v_mul_lo_u32 v19, v12, v7 -; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v4, v12, 0 -; GFX9-NEXT: v_mov_b32_e32 v17, 0 -; GFX9-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v12, v6, 0 -; GFX9-NEXT: v_mov_b32_e32 v16, v8 -; GFX9-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v5, v12, v[16:17] -; GFX9-NEXT: v_mul_lo_u32 v18, v13, v6 -; GFX9-NEXT: v_mul_lo_u32 v16, v15, v4 -; GFX9-NEXT: v_mov_b32_e32 v6, v12 -; GFX9-NEXT: v_mov_b32_e32 v12, v17 -; GFX9-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v4, v13, v[11:12] -; GFX9-NEXT: v_add3_u32 v10, v10, v19, v18 -; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v14, v4, v[9:10] -; GFX9-NEXT: v_mov_b32_e32 v4, v12 -; GFX9-NEXT: v_mul_lo_u32 v10, v14, v5 -; GFX9-NEXT: v_add_co_u32_e32 v14, vcc, v6, v4 -; GFX9-NEXT: v_addc_co_u32_e64 v15, s[4:5], 0, 0, vcc -; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v5, v13, v[14:15] -; GFX9-NEXT: v_add3_u32 v6, v16, v9, v10 -; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v6, vcc -; GFX9-NEXT: v_mov_b32_e32 v6, v11 -; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v7 -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v4, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v5, vcc -; GFX9-NEXT: s_setpc_b64 s[30:31] -; -; GFX9-O0-LABEL: v_urem_i128_vv: -; GFX9-O0: ; %bb.0: ; %_udiv-special-cases -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] -; GFX9-O0-NEXT: ; implicit-def: $vgpr8 : SGPR spill to VGPR lane -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v6 -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v13, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v1 -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v0 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v2 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v3 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v12 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v12 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v2 -; GFX9-O0-NEXT: v_or_b32_e64 v3, v8, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 -; GFX9-O0-NEXT: v_or_b32_e64 v1, v5, v6 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 -; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 0 -; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 1 -; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[1:2], s[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v14 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 -; GFX9-O0-NEXT: v_or_b32_e64 v15, v4, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 -; GFX9-O0-NEXT: v_or_b32_e64 v9, v3, v1 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v15 -; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[9:10], s[6:7] -; GFX9-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9] -; GFX9-O0-NEXT: v_ffbh_u32_e64 v6, v6 -; GFX9-O0-NEXT: s_mov_b32 s9, 32 -; GFX9-O0-NEXT: v_add_u32_e64 v6, v6, s9 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v7, v7 -; GFX9-O0-NEXT: v_min_u32_e64 v6, v6, v7 -; GFX9-O0-NEXT: s_mov_b32 s8, 0 -; GFX9-O0-NEXT: ; implicit-def: $sgpr10 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, s8 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v7 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v5 -; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v8, v8 -; GFX9-O0-NEXT: v_min_u32_e64 v15, v5, v8 -; GFX9-O0-NEXT: ; implicit-def: $sgpr10 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v5 -; GFX9-O0-NEXT: s_mov_b64 s[10:11], 64 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v15 -; GFX9-O0-NEXT: s_mov_b32 s12, s10 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16 -; GFX9-O0-NEXT: s_mov_b32 s14, s11 -; GFX9-O0-NEXT: v_add_co_u32_e64 v8, s[12:13], v8, s12 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, s14 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[12:13], v5, v9, s[12:13] -; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 -; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[6:7] -; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[12:13], v[11:12], s[12:13] -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v10, s[12:13] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v8 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v9, v6, v7, s[12:13] -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v1 -; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v6, v2 -; GFX9-O0-NEXT: v_min_u32_e64 v6, v5, v6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v7 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v3 -; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v11, v4 -; GFX9-O0-NEXT: v_min_u32_e64 v15, v5, v11 -; GFX9-O0-NEXT: ; implicit-def: $sgpr9 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v15 -; GFX9-O0-NEXT: s_mov_b32 s8, s10 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16 -; GFX9-O0-NEXT: s_mov_b32 s10, s11 -; GFX9-O0-NEXT: v_add_co_u32_e64 v11, s[8:9], v11, s8 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, s10 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[8:9], v5, v12, s[8:9] -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 -; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] -; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[8:9], v[13:14], s[8:9] -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[8:9] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[8:9] -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 -; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 -; GFX9-O0-NEXT: s_mov_b32 s10, s6 -; GFX9-O0-NEXT: s_mov_b32 s11, s7 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v5, vcc, v5, v8 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v9, vcc, v6, v7, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v7, s10 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, s10 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v8, vcc, v6, v7, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v7, s11 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, s11 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v6, v7, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v9 -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v8 -; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 -; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] -; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[8:9], s[8:9] -; GFX9-O0-NEXT: s_mov_b64 s[12:13], 0x7f -; GFX9-O0-NEXT: s_mov_b64 s[14:15], s[12:13] -; GFX9-O0-NEXT: v_cmp_gt_u64_e64 s[14:15], v[5:6], s[14:15] -; GFX9-O0-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[14:15] -; GFX9-O0-NEXT: s_mov_b64 s[14:15], s[6:7] -; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[14:15], v[8:9], s[14:15] -; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[14:15] -; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, v7, v10, s[8:9] -; GFX9-O0-NEXT: v_and_b32_e64 v7, 1, v7 -; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[8:9], v7, 1 -; GFX9-O0-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9] -; GFX9-O0-NEXT: s_mov_b64 s[4:5], -1 -; GFX9-O0-NEXT: s_xor_b64 s[4:5], s[8:9], s[4:5] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 -; GFX9-O0-NEXT: s_mov_b32 s14, s13 -; GFX9-O0-NEXT: v_xor_b32_e64 v7, v7, s14 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13 -; GFX9-O0-NEXT: v_xor_b32_e64 v5, v5, s12 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v9 -; GFX9-O0-NEXT: v_or_b32_e64 v7, v7, v10 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v8 -; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v6 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7 -; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[5:6], s[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v2, v5, s[8:9] -; GFX9-O0-NEXT: v_mov_b32_e32 v2, s10 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v1, v1, v2, s[8:9] -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v4, v5, s[8:9] -; GFX9-O0-NEXT: v_mov_b32_e32 v4, s10 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[8:9] -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; implicit-def: $sgpr8 -; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v5 -; GFX9-O0-NEXT: s_and_b64 s[6:7], s[4:5], s[6:7] -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 s[4:5], exec -; GFX9-O0-NEXT: v_writelane_b32 v0, s4, 2 -; GFX9-O0-NEXT: v_writelane_b32 v0, s5, 3 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7] -; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] -; GFX9-O0-NEXT: s_cbranch_execz .LBB3_3 -; GFX9-O0-NEXT: s_branch .LBB3_8 -; GFX9-O0-NEXT: .LBB3_1: ; %Flow -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s4, v0, 4 -; GFX9-O0-NEXT: v_readlane_b32 s5, v0, 5 -; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX9-O0-NEXT: ; %bb.2: ; %Flow -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_waitcnt vmcnt(6) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_branch .LBB3_5 -; GFX9-O0-NEXT: .LBB3_3: ; %Flow2 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s4, v4, 2 -; GFX9-O0-NEXT: v_readlane_b32 s5, v4, 3 -; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_branch .LBB3_9 -; GFX9-O0-NEXT: .LBB3_4: ; %udiv-loop-exit -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b32 s4, 1 -; GFX9-O0-NEXT: s_waitcnt vmcnt(2) -; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[0:1] -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_lshlrev_b64 v[9:10], s4, v[9:10] -; GFX9-O0-NEXT: s_mov_b32 s4, 63 -; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1] -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v8 -; GFX9-O0-NEXT: v_or3_b32 v4, v4, v11, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v9 -; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v7 -; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v6 -; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5 -; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v4 -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_branch .LBB3_3 -; GFX9-O0-NEXT: .LBB3_5: ; %Flow1 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s4, v8, 6 -; GFX9-O0-NEXT: v_readlane_b32 s5, v8, 7 -; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_branch .LBB3_4 -; GFX9-O0-NEXT: .LBB3_6: ; %udiv-do-while -; GFX9-O0-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s6, v16, 8 -; GFX9-O0-NEXT: v_readlane_b32 s7, v16, 9 -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b32 s4, 63 -; GFX9-O0-NEXT: s_waitcnt vmcnt(16) -; GFX9-O0-NEXT: v_lshrrev_b64 v[29:30], s4, v[2:3] -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v30 -; GFX9-O0-NEXT: s_mov_b32 s5, 1 -; GFX9-O0-NEXT: v_lshlrev_b64 v[23:24], s5, v[23:24] -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v24 -; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v29 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v23 -; GFX9-O0-NEXT: v_or_b32_e64 v23, v5, v10 -; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v24, v4 -; GFX9-O0-NEXT: v_lshlrev_b64 v[29:30], s5, v[2:3] -; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], s4, v[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v30 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 -; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v29 -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec -; GFX9-O0-NEXT: v_or_b32_e64 v4, v3, v4 -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v2 -; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s5, v[0:1] -; GFX9-O0-NEXT: v_lshlrev_b64 v[29:30], s5, v[6:7] -; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v30 -; GFX9-O0-NEXT: s_waitcnt vmcnt(10) -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v28 -; GFX9-O0-NEXT: v_or3_b32 v6, v6, v7, v10 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v29 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v27 -; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v7 -; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3 -; GFX9-O0-NEXT: s_waitcnt vmcnt(8) -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v26 -; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v25 -; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v4 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v23 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v24 -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_mov_b32_e32 v13, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v14 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v15 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v13, vcc, v13, v6 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v12, vcc, v12, v10, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v11, vcc, v11, v4, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v7, v5, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v7 -; GFX9-O0-NEXT: v_ashrrev_i64 v[13:14], s4, v[11:12] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v14 -; GFX9-O0-NEXT: s_mov_b64 s[4:5], 1 -; GFX9-O0-NEXT: s_mov_b32 s8, s5 -; GFX9-O0-NEXT: v_and_b32_e64 v12, v7, s8 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v13 -; GFX9-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5 -; GFX9-O0-NEXT: v_and_b32_e64 v14, v11, s4 -; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v13, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v23, v22 -; GFX9-O0-NEXT: v_and_b32_e64 v23, v7, v23 -; GFX9-O0-NEXT: v_and_b32_e64 v21, v11, v21 -; GFX9-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v22, v23 -; GFX9-O0-NEXT: v_mov_b32_e32 v23, v20 -; GFX9-O0-NEXT: v_and_b32_e64 v7, v7, v23 -; GFX9-O0-NEXT: v_and_b32_e64 v23, v11, v19 -; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v24, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v23 -; GFX9-O0-NEXT: v_mov_b32_e32 v19, v24 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v21 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v22 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v6, vcc, v6, v20 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v10, vcc, v10, v19, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v4, vcc, v4, v11, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v5, v7, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v10 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v8 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 killed $vgpr8_vgpr9 killed $exec -; GFX9-O0-NEXT: s_mov_b64 s[8:9], -1 -; GFX9-O0-NEXT: s_mov_b32 s5, s8 -; GFX9-O0-NEXT: s_mov_b32 s4, s9 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v18 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, s5 -; GFX9-O0-NEXT: v_add_co_u32_e32 v20, vcc, v11, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, s4 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v9, vcc, v9, v11, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v11, s5 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v17, vcc, v10, v11, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v10, s4 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v8, vcc, v8, v10, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr20 killed $vgpr20 def $vgpr20_vgpr21 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v21, v9 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v8 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v18 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v20 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v21 -; GFX9-O0-NEXT: v_mov_b32_e32 v22, v18 -; GFX9-O0-NEXT: v_mov_b32_e32 v19, v21 -; GFX9-O0-NEXT: v_or_b32_e64 v19, v19, v22 -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, v20 -; GFX9-O0-NEXT: v_or_b32_e64 v17, v17, v18 -; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v19 -; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[17:18], v[12:13] -; GFX9-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, v2 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, v0 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v15 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, v14 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, v12 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5] -; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 4 -; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 5 -; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5] -; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 8 -; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 9 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX9-O0-NEXT: s_cbranch_execnz .LBB3_6 -; GFX9-O0-NEXT: s_branch .LBB3_1 -; GFX9-O0-NEXT: .LBB3_7: ; %udiv-preheader -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_waitcnt vmcnt(9) -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], v4, v[21:22] -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7 -; GFX9-O0-NEXT: s_mov_b32 s6, 64 -; GFX9-O0-NEXT: v_sub_u32_e64 v12, s6, v4 -; GFX9-O0-NEXT: v_lshlrev_b64 v[23:24], v12, v[19:20] -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v24 -; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v12 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v23 -; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v7 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v7 -; GFX9-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v4, s6 -; GFX9-O0-NEXT: v_sub_u32_e64 v5, v4, s6 -; GFX9-O0-NEXT: v_lshrrev_b64 v[23:24], v5, v[19:20] -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v24 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v12, s[4:5] -; GFX9-O0-NEXT: s_mov_b32 s6, 0 -; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v4, s6 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v22 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v12, s[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v23 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v21 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[6:7] -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[19:20] -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v5 -; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 -; GFX9-O0-NEXT: s_mov_b32 s8, s7 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, s8 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v12, v12, v15, s[4:5] -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4 -; GFX9-O0-NEXT: s_mov_b32 s8, s6 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[4:5] -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v14 -; GFX9-O0-NEXT: s_mov_b64 s[8:9], -1 -; GFX9-O0-NEXT: s_mov_b32 s5, s8 -; GFX9-O0-NEXT: s_mov_b32 s4, s9 -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v13, v18 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, s5 -; GFX9-O0-NEXT: v_add_co_u32_e32 v12, vcc, v12, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, s4 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v17, vcc, v15, v17, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v15, s5 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v14, vcc, v14, v15, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v15, s4 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v13, vcc, v13, v15, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v13 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v13, v17 -; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] -; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v15, s9 -; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8 -; GFX9-O0-NEXT: v_mov_b32_e32 v13, s7 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, s6 -; GFX9-O0-NEXT: v_writelane_b32 v16, s4, 8 -; GFX9-O0-NEXT: v_writelane_b32 v16, s5, 9 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_branch .LBB3_6 -; GFX9-O0-NEXT: .LBB3_8: ; %udiv-bb1 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 s[6:7], 1 -; GFX9-O0-NEXT: s_mov_b32 s5, s6 -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v1 -; GFX9-O0-NEXT: s_mov_b32 s4, s7 -; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 -; GFX9-O0-NEXT: s_mov_b32 s8, s6 -; GFX9-O0-NEXT: s_mov_b32 s9, s7 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s5 -; GFX9-O0-NEXT: v_add_co_u32_e32 v9, vcc, v4, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s4 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v5, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s9 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v5, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v2 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b32 s4, 0x7f -; GFX9-O0-NEXT: v_sub_u32_e64 v3, s4, v4 -; GFX9-O0-NEXT: v_lshlrev_b64 v[5:6], v3, v[11:12] -; GFX9-O0-NEXT: v_mov_b32_e32 v13, v6 -; GFX9-O0-NEXT: s_mov_b32 s4, 64 -; GFX9-O0-NEXT: v_sub_u32_e64 v14, s4, v3 -; GFX9-O0-NEXT: v_lshrrev_b64 v[14:15], v14, v[7:8] -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v15 -; GFX9-O0-NEXT: v_or_b32_e64 v13, v13, v16 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v14 -; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v6 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v6 -; GFX9-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v3, s4 -; GFX9-O0-NEXT: s_mov_b32 s10, 63 -; GFX9-O0-NEXT: v_sub_u32_e64 v4, s10, v4 -; GFX9-O0-NEXT: v_lshlrev_b64 v[13:14], v4, v[7:8] -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v15, s[4:5] -; GFX9-O0-NEXT: s_mov_b32 s10, 0 -; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[10:11], v3, s10 -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v15, s[10:11] -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v13 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[4:5] -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[10:11] -; GFX9-O0-NEXT: ; implicit-def: $sgpr10 -; GFX9-O0-NEXT: ; implicit-def: $sgpr10 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v4 -; GFX9-O0-NEXT: v_lshlrev_b64 v[7:8], v3, v[7:8] -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v8 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, s9 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[4:5] -; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr7_vgpr8 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, v4, v7, s[4:5] -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v3 -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v10 -; GFX9-O0-NEXT: v_or_b32_e64 v3, v3, v4 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v9 -; GFX9-O0-NEXT: v_or_b32_e64 v1, v1, v2 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[4:5], v[1:2], s[6:7] -; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v3, s8 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, s9 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, s6 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, s7 -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 s[6:7], exec -; GFX9-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5] -; GFX9-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7] -; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 6 -; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 7 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] -; GFX9-O0-NEXT: s_cbranch_execz .LBB3_5 -; GFX9-O0-NEXT: s_branch .LBB3_7 -; GFX9-O0-NEXT: .LBB3_9: ; %udiv-end -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b32 s4, 32 -; GFX9-O0-NEXT: s_waitcnt vmcnt(2) -; GFX9-O0-NEXT: v_lshrrev_b64 v[2:3], s4, v[7:8] -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v13 -; GFX9-O0-NEXT: v_mul_lo_u32 v5, v6, v2 -; GFX9-O0-NEXT: v_lshrrev_b64 v[13:14], s4, v[13:14] -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13 -; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr7_vgpr8 killed $exec -; GFX9-O0-NEXT: v_mul_lo_u32 v3, v7, v3 -; GFX9-O0-NEXT: v_mad_u64_u32 v[13:14], s[6:7], v7, v2, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v14 -; GFX9-O0-NEXT: v_add3_u32 v2, v2, v3, v5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 -; GFX9-O0-NEXT: v_lshlrev_b64 v[17:18], s4, v[2:3] -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v18 -; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 killed $vgpr13_vgpr14 killed $exec -; GFX9-O0-NEXT: s_mov_b32 s5, 0 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v14 -; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13 -; GFX9-O0-NEXT: v_or_b32_e64 v13, v3, v5 -; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v2 -; GFX9-O0-NEXT: v_lshrrev_b64 v[2:3], s4, v[15:16] -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v11 -; GFX9-O0-NEXT: v_mul_lo_u32 v3, v2, v8 -; GFX9-O0-NEXT: v_lshrrev_b64 v[11:12], s4, v[11:12] -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 killed $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v15 -; GFX9-O0-NEXT: v_mul_lo_u32 v11, v11, v5 -; GFX9-O0-NEXT: v_mad_u64_u32 v[15:16], s[6:7], v2, v5, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v16 -; GFX9-O0-NEXT: v_add3_u32 v2, v2, v3, v11 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, s6 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[2:3] -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v3 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 killed $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v16 -; GFX9-O0-NEXT: v_or_b32_e64 v11, v11, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v15 -; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v14 -; GFX9-O0-NEXT: v_add_co_u32_e64 v13, s[6:7], v11, v12 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v2, s[6:7], v2, v3, s[6:7] -; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v2 -; GFX9-O0-NEXT: v_mad_u64_u32 v[15:16], s[6:7], v8, v6, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v15 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v16 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, s6 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v12 -; GFX9-O0-NEXT: v_lshlrev_b64 v[15:16], s4, v[15:16] -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v16 -; GFX9-O0-NEXT: v_or_b32_e64 v11, v11, v12 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v15 -; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_mad_u64_u32 v[15:16], s[6:7], v8, v7, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v15 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v8 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v16 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, s6 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v17 -; GFX9-O0-NEXT: v_lshlrev_b64 v[15:16], s4, v[15:16] -; GFX9-O0-NEXT: v_mov_b32_e32 v17, v16 -; GFX9-O0-NEXT: v_or_b32_e64 v8, v8, v17 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 killed $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v15 -; GFX9-O0-NEXT: v_or_b32_e64 v19, v11, v12 -; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v8 -; GFX9-O0-NEXT: v_mad_u64_u32 v[11:12], s[6:7], v5, v7, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v17, v12 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v19 -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v20 -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v18 -; GFX9-O0-NEXT: v_add_co_u32_e64 v7, s[6:7], v7, v16 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v15, s[6:7], v8, v15, s[6:7] -; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v15 -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v8 -; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0xffffffff -; GFX9-O0-NEXT: s_mov_b32 s8, s7 -; GFX9-O0-NEXT: v_and_b32_e64 v15, v15, s8 -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v7 -; GFX9-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 killed $sgpr6_sgpr7 -; GFX9-O0-NEXT: v_and_b32_e64 v17, v16, s6 -; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v18, v15 -; GFX9-O0-NEXT: v_mad_u64_u32 v[15:16], s[6:7], v5, v6, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v19, v15 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v20 -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v16 -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr7 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, s6 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v6 -; GFX9-O0-NEXT: v_lshlrev_b64 v[15:16], s4, v[15:16] -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v16 -; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v19 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 killed $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_or_b32_e64 v19, v6, v15 -; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v20, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v19 -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v20 -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v18 -; GFX9-O0-NEXT: v_add_co_u32_e64 v5, s[6:7], v5, v16 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v15, s[6:7], v6, v15, s[6:7] -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v15 -; GFX9-O0-NEXT: v_lshrrev_b64 v[17:18], s4, v[5:6] -; GFX9-O0-NEXT: v_lshrrev_b64 v[7:8], s4, v[7:8] -; GFX9-O0-NEXT: v_mov_b32_e32 v15, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v17 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v8 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v18 -; GFX9-O0-NEXT: v_add_co_u32_e64 v15, s[6:7], v15, v16 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v7, v8, s[6:7] -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v15 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v16 -; GFX9-O0-NEXT: v_add_co_u32_e64 v15, s[6:7], v7, v8 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v2, s[6:7], v2, v3, s[6:7] -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v15 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v16 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v14 -; GFX9-O0-NEXT: v_add_co_u32_e64 v2, s[6:7], v2, v8 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v3, v7, s[6:7] -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v7 -; GFX9-O0-NEXT: v_lshlrev_b64 v[6:7], s4, v[5:6] -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v7 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 killed $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: ; implicit-def: $sgpr6 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s5 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 -; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v8 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 -; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v7 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v10 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v7, vcc, v7, v8 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v6, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v5, vcc, v3, v5, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v2, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v1 -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr5 -; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5 -; GFX9-O0-NEXT: v_lshrrev_b64 v[7:8], s4, v[7:8] -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 -; GFX9-O0-NEXT: v_lshrrev_b64 v[5:6], s4, v[5:6] -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 -; GFX9-O0-NEXT: ; kill: killed $vgpr4 -; GFX9-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_nop 0 -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: s_setpc_b64 s[30:31] - %div = urem i128 %lhs, %rhs - ret i128 %div -} - define i128 @v_sdiv_i128_v_pow2k(i128 %lhs) { ; GFX9-LABEL: v_sdiv_i128_v_pow2k: ; GFX9: ; %bb.0: @@ -5246,106 +2392,6 @@ define i128 @v_sdiv_i128_v_pow2k(i128 %lhs) { ret i128 %div } -define i128 @v_srem_i128_v_pow2k(i128 %lhs) { -; GFX9-LABEL: v_srem_i128_v_pow2k: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_ashrrev_i32_e32 v4, 31, v3 -; GFX9-NEXT: v_mov_b32_e32 v5, v4 -; GFX9-NEXT: v_lshrrev_b64 v[4:5], 31, v[4:5] -; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v0, v4 -; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v1, v5, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v2, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v3, vcc -; GFX9-NEXT: v_and_b32_e32 v4, -2, v4 -; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, 0, v0 -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v4, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v5, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v6, vcc -; GFX9-NEXT: s_setpc_b64 s[30:31] -; -; GFX9-O0-LABEL: v_srem_i128_v_pow2k: -; GFX9-O0: ; %bb.0: -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v1 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v2 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 -; GFX9-O0-NEXT: s_mov_b32 s4, 63 -; GFX9-O0-NEXT: v_ashrrev_i64 v[6:7], s4, v[6:7] -; GFX9-O0-NEXT: s_mov_b32 s4, 31 -; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], s4, v[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v7 -; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 -; GFX9-O0-NEXT: s_mov_b32 s5, s6 -; GFX9-O0-NEXT: s_mov_b32 s4, s7 -; GFX9-O0-NEXT: v_add_co_u32_e32 v6, vcc, v5, v4 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v4, vcc, v0, v2, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v2, s5 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v8, vcc, v3, v2, vcc -; GFX9-O0-NEXT: v_mov_b32_e32 v2, s4 -; GFX9-O0-NEXT: v_addc_co_u32_e32 v2, vcc, v1, v2, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v4 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v7 -; GFX9-O0-NEXT: s_mov_b32 s6, -2 -; GFX9-O0-NEXT: s_mov_b32 s4, 0 -; GFX9-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5 -; GFX9-O0-NEXT: s_mov_b32 s5, s6 -; GFX9-O0-NEXT: s_mov_b32 s6, s5 -; GFX9-O0-NEXT: v_and_b32_e64 v4, v4, s6 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5 -; GFX9-O0-NEXT: v_and_b32_e64 v9, v6, s4 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v4 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v8 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v9 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v5, vcc, v5, v7 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v6, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v4, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v2, vcc -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v1 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v0 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: s_mov_b32 s4, 32 -; GFX9-O0-NEXT: v_lshrrev_b64 v[5:6], s4, v[5:6] -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v5 -; GFX9-O0-NEXT: v_lshrrev_b64 v[3:4], s4, v[3:4] -; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr3_vgpr4 killed $exec -; GFX9-O0-NEXT: s_setpc_b64 s[30:31] - %div = srem i128 %lhs, 8589934592 - ret i128 %div -} - define i128 @v_udiv_i128_v_pow2k(i128 %lhs) { ; GFX9-LABEL: v_udiv_i128_v_pow2k: ; GFX9: ; %bb.0: @@ -5392,55 +2438,6 @@ define i128 @v_udiv_i128_v_pow2k(i128 %lhs) { ret i128 %div } -define i128 @v_urem_i128_v_pow2k(i128 %lhs) { -; GFX9-LABEL: v_urem_i128_v_pow2k: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_and_b32_e32 v1, 1, v1 -; GFX9-NEXT: v_mov_b32_e32 v2, 0 -; GFX9-NEXT: v_mov_b32_e32 v3, 0 -; GFX9-NEXT: s_setpc_b64 s[30:31] -; -; GFX9-O0-LABEL: v_urem_i128_v_pow2k: -; GFX9-O0: ; %bb.0: -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v1 -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr1 killed $exec -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v2 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 -; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 -; GFX9-O0-NEXT: s_mov_b32 s6, 1 -; GFX9-O0-NEXT: s_mov_b32 s4, -1 -; GFX9-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5 -; GFX9-O0-NEXT: s_mov_b32 s5, s6 -; GFX9-O0-NEXT: s_mov_b32 s6, s5 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v1 -; GFX9-O0-NEXT: v_and_b32_e64 v3, v2, s6 -; GFX9-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5 -; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec -; GFX9-O0-NEXT: v_and_b32_e64 v1, v0, s4 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: s_mov_b32 s4, 32 -; GFX9-O0-NEXT: v_lshrrev_b64 v[1:2], s4, v[1:2] -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, 0 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: s_setpc_b64 s[30:31] - %div = urem i128 %lhs, 8589934592 - ret i128 %div -} - ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; GFX9-SDAG: {{.*}} ; GFX9-SDAG-O0: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll index 09898f1..38640a1 100644 --- a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll +++ b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll @@ -152,7 +152,7 @@ define amdgpu_kernel void @constant_fold_fmax_f32_p0_n0(ptr addrspace(1) %out) # ; GCN-LABEL: {{^}}constant_fold_fmax_f32_n0_p0: ; GCN-NOT: v_max_f32_e32 -; GCN: v_bfrev_b32_e32 [[REG:v[0-9]+]], 1{{$}} +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0{{$}} ; GCN: buffer_store_dword [[REG]] define amdgpu_kernel void @constant_fold_fmax_f32_n0_p0(ptr addrspace(1) %out) #0 { %val = call float @llvm.maxnum.f32(float -0.0, float 0.0) diff --git a/llvm/test/CodeGen/AMDGPU/fminnum.ll b/llvm/test/CodeGen/AMDGPU/fminnum.ll index 844d26a..65b3118 100644 --- a/llvm/test/CodeGen/AMDGPU/fminnum.ll +++ b/llvm/test/CodeGen/AMDGPU/fminnum.ll @@ -150,7 +150,7 @@ define amdgpu_kernel void @constant_fold_fmin_f32_p0_p0(ptr addrspace(1) %out) # ; GCN-LABEL: {{^}}constant_fold_fmin_f32_p0_n0: ; GCN-NOT: v_min_f32_e32 -; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0 +; GCN: v_bfrev_b32_e32 [[REG:v[0-9]+]], 1{{$}} ; GCN: buffer_store_dword [[REG]] define amdgpu_kernel void @constant_fold_fmin_f32_p0_n0(ptr addrspace(1) %out) #0 { %val = call float @llvm.minnum.f32(float 0.0, float -0.0) diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll new file mode 100644 index 0000000..6ba66cc --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll @@ -0,0 +1,3014 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s +; RUN: llc -O0 -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9-O0,GFX9-SDAG-O0 %s + +; FIXME: GlobalISel missing the power-of-2 cases in legalization. https://github.com/llvm/llvm-project/issues/80671 +; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9,GFX9 %s +; xUN: llc -O0 -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9-O0,GFX9-O0 %s}} + +define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { +; GFX9-LABEL: v_srem_i128_vv: +; GFX9: ; %bb.0: ; %_udiv-special-cases +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_ashrrev_i32_e32 v20, 31, v3 +; GFX9-NEXT: v_xor_b32_e32 v0, v0, v20 +; GFX9-NEXT: v_xor_b32_e32 v10, v2, v20 +; GFX9-NEXT: v_xor_b32_e32 v1, v1, v20 +; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v20 +; GFX9-NEXT: v_xor_b32_e32 v9, v3, v20 +; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v20, vcc +; GFX9-NEXT: v_ashrrev_i32_e32 v8, 31, v7 +; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v10, v20, vcc +; GFX9-NEXT: v_xor_b32_e32 v4, v4, v8 +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v9, v20, vcc +; GFX9-NEXT: v_xor_b32_e32 v5, v5, v8 +; GFX9-NEXT: v_sub_co_u32_e32 v23, vcc, v4, v8 +; GFX9-NEXT: v_xor_b32_e32 v6, v6, v8 +; GFX9-NEXT: v_subb_co_u32_e32 v21, vcc, v5, v8, vcc +; GFX9-NEXT: v_xor_b32_e32 v7, v7, v8 +; GFX9-NEXT: v_subb_co_u32_e32 v4, vcc, v6, v8, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v8, vcc +; GFX9-NEXT: v_or_b32_e32 v7, v21, v5 +; GFX9-NEXT: v_or_b32_e32 v6, v23, v4 +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7] +; GFX9-NEXT: v_or_b32_e32 v7, v3, v1 +; GFX9-NEXT: v_or_b32_e32 v6, v2, v0 +; GFX9-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[6:7] +; GFX9-NEXT: v_ffbh_u32_e32 v6, v4 +; GFX9-NEXT: v_add_u32_e32 v6, 32, v6 +; GFX9-NEXT: v_ffbh_u32_e32 v7, v5 +; GFX9-NEXT: v_min_u32_e32 v6, v6, v7 +; GFX9-NEXT: v_ffbh_u32_e32 v7, v23 +; GFX9-NEXT: v_add_u32_e32 v7, 32, v7 +; GFX9-NEXT: v_ffbh_u32_e32 v8, v21 +; GFX9-NEXT: v_min_u32_e32 v7, v7, v8 +; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, 64, v7 +; GFX9-NEXT: v_addc_co_u32_e64 v8, s[6:7], 0, 0, vcc +; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] +; GFX9-NEXT: v_ffbh_u32_e32 v9, v1 +; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc +; GFX9-NEXT: v_ffbh_u32_e32 v7, v0 +; GFX9-NEXT: v_add_u32_e32 v7, 32, v7 +; GFX9-NEXT: v_min_u32_e32 v7, v7, v9 +; GFX9-NEXT: v_ffbh_u32_e32 v9, v2 +; GFX9-NEXT: v_add_u32_e32 v9, 32, v9 +; GFX9-NEXT: v_ffbh_u32_e32 v10, v3 +; GFX9-NEXT: v_min_u32_e32 v9, v9, v10 +; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, 0, vcc +; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, 64, v9 +; GFX9-NEXT: v_addc_co_u32_e64 v10, s[6:7], 0, 0, vcc +; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f +; GFX9-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc +; GFX9-NEXT: v_sub_co_u32_e32 v6, vcc, v6, v7 +; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v8, v10, vcc +; GFX9-NEXT: v_mov_b32_e32 v9, 0 +; GFX9-NEXT: v_subbrev_co_u32_e32 v8, vcc, 0, v9, vcc +; GFX9-NEXT: v_subbrev_co_u32_e32 v9, vcc, 0, v9, vcc +; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7] +; GFX9-NEXT: v_or_b32_e32 v13, v7, v9 +; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc +; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9] +; GFX9-NEXT: v_mov_b32_e32 v22, v20 +; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] +; GFX9-NEXT: v_cndmask_b32_e32 v10, v11, v10, vcc +; GFX9-NEXT: v_and_b32_e32 v10, 1, v10 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10 +; GFX9-NEXT: v_xor_b32_e32 v10, 0x7f, v6 +; GFX9-NEXT: v_or_b32_e32 v12, v10, v8 +; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], vcc +; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13] +; GFX9-NEXT: s_xor_b64 s[6:7], s[4:5], -1 +; GFX9-NEXT: v_cndmask_b32_e64 v11, v1, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v12, v0, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v10, v3, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v13, v2, 0, s[4:5] +; GFX9-NEXT: s_and_b64 s[4:5], s[6:7], vcc +; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] +; GFX9-NEXT: s_cbranch_execz .LBB0_6 +; GFX9-NEXT: ; %bb.1: ; %udiv-bb1 +; GFX9-NEXT: v_add_co_u32_e32 v24, vcc, 1, v6 +; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v7, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v26, vcc, 0, v8, vcc +; GFX9-NEXT: v_sub_u32_e32 v13, 0x7f, v6 +; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, 0, v9, vcc +; GFX9-NEXT: v_sub_u32_e32 v11, 64, v13 +; GFX9-NEXT: v_or_b32_e32 v8, v25, v27 +; GFX9-NEXT: v_or_b32_e32 v7, v24, v26 +; GFX9-NEXT: v_lshlrev_b64 v[9:10], v13, v[0:1] +; GFX9-NEXT: v_lshrrev_b64 v[11:12], v11, v[2:3] +; GFX9-NEXT: v_sub_u32_e32 v6, 63, v6 +; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[7:8] +; GFX9-NEXT: v_lshlrev_b64 v[6:7], v6, v[2:3] +; GFX9-NEXT: v_or_b32_e32 v8, v10, v12 +; GFX9-NEXT: v_or_b32_e32 v9, v9, v11 +; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v13 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v13 +; GFX9-NEXT: v_lshlrev_b64 v[12:13], v13, v[2:3] +; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v8, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[4:5] +; GFX9-NEXT: v_mov_b32_e32 v8, 0 +; GFX9-NEXT: v_mov_b32_e32 v10, 0 +; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v1, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v0, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v13, 0, v13, s[4:5] +; GFX9-NEXT: v_mov_b32_e32 v9, 0 +; GFX9-NEXT: v_mov_b32_e32 v11, 0 +; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, v12, s[4:5] +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_xor_b64 s[6:7], exec, s[4:5] +; GFX9-NEXT: s_cbranch_execz .LBB0_5 +; GFX9-NEXT: ; %bb.2: ; %udiv-preheader +; GFX9-NEXT: v_sub_u32_e32 v10, 64, v24 +; GFX9-NEXT: v_lshrrev_b64 v[8:9], v24, v[2:3] +; GFX9-NEXT: v_lshlrev_b64 v[10:11], v10, v[0:1] +; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v24 +; GFX9-NEXT: v_or_b32_e32 v10, v8, v10 +; GFX9-NEXT: v_subrev_u32_e32 v8, 64, v24 +; GFX9-NEXT: v_or_b32_e32 v11, v9, v11 +; GFX9-NEXT: v_lshrrev_b64 v[8:9], v8, v[0:1] +; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v24 +; GFX9-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v15, v9, v3, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v10, v8, v10, vcc +; GFX9-NEXT: v_lshrrev_b64 v[8:9], v24, v[0:1] +; GFX9-NEXT: v_cndmask_b32_e64 v14, v10, v2, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v17, 0, v9, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v16, 0, v8, vcc +; GFX9-NEXT: v_add_co_u32_e32 v28, vcc, -1, v23 +; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v21, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v30, vcc, -1, v4, vcc +; GFX9-NEXT: v_mov_b32_e32 v18, 0 +; GFX9-NEXT: v_mov_b32_e32 v10, 0 +; GFX9-NEXT: v_addc_co_u32_e32 v31, vcc, -1, v5, vcc +; GFX9-NEXT: s_mov_b64 s[4:5], 0 +; GFX9-NEXT: v_mov_b32_e32 v19, 0 +; GFX9-NEXT: v_mov_b32_e32 v11, 0 +; GFX9-NEXT: v_mov_b32_e32 v9, 0 +; GFX9-NEXT: .LBB0_3: ; %udiv-do-while +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_lshrrev_b32_e32 v32, 31, v15 +; GFX9-NEXT: v_lshlrev_b64 v[14:15], 1, v[14:15] +; GFX9-NEXT: v_lshrrev_b32_e32 v33, 31, v7 +; GFX9-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7] +; GFX9-NEXT: v_lshrrev_b32_e32 v8, 31, v13 +; GFX9-NEXT: v_lshlrev_b64 v[16:17], 1, v[16:17] +; GFX9-NEXT: v_or_b32_e32 v14, v14, v33 +; GFX9-NEXT: v_or3_b32 v6, v6, v8, v10 +; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v28, v14 +; GFX9-NEXT: v_or_b32_e32 v16, v16, v32 +; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, v29, v15, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, v30, v16, vcc +; GFX9-NEXT: v_lshlrev_b64 v[12:13], 1, v[12:13] +; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, v31, v17, vcc +; GFX9-NEXT: v_ashrrev_i32_e32 v8, 31, v8 +; GFX9-NEXT: v_or_b32_e32 v12, v18, v12 +; GFX9-NEXT: v_and_b32_e32 v18, v8, v23 +; GFX9-NEXT: v_or_b32_e32 v13, v19, v13 +; GFX9-NEXT: v_and_b32_e32 v19, v8, v21 +; GFX9-NEXT: v_sub_co_u32_e32 v14, vcc, v14, v18 +; GFX9-NEXT: v_and_b32_e32 v32, v8, v4 +; GFX9-NEXT: v_subb_co_u32_e32 v15, vcc, v15, v19, vcc +; GFX9-NEXT: v_and_b32_e32 v33, v8, v5 +; GFX9-NEXT: v_subb_co_u32_e32 v16, vcc, v16, v32, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v17, vcc, v17, v33, vcc +; GFX9-NEXT: v_add_co_u32_e32 v24, vcc, -1, v24 +; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v25, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v26, vcc, -1, v26, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, -1, v27, vcc +; GFX9-NEXT: v_or_b32_e32 v18, v24, v26 +; GFX9-NEXT: v_or_b32_e32 v19, v25, v27 +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[18:19] +; GFX9-NEXT: v_and_b32_e32 v8, 1, v8 +; GFX9-NEXT: v_mov_b32_e32 v19, v9 +; GFX9-NEXT: v_or3_b32 v7, v7, 0, v11 +; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX9-NEXT: v_mov_b32_e32 v18, v8 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX9-NEXT: s_cbranch_execnz .LBB0_3 +; GFX9-NEXT: ; %bb.4: ; %Flow +; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-NEXT: .LBB0_5: ; %Flow2 +; GFX9-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX9-NEXT: v_lshlrev_b64 v[14:15], 1, v[12:13] +; GFX9-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7] +; GFX9-NEXT: v_lshrrev_b32_e32 v12, 31, v13 +; GFX9-NEXT: v_or3_b32 v11, v7, 0, v11 +; GFX9-NEXT: v_or3_b32 v12, v6, v12, v10 +; GFX9-NEXT: v_or_b32_e32 v10, v9, v15 +; GFX9-NEXT: v_or_b32_e32 v13, v8, v14 +; GFX9-NEXT: .LBB0_6: ; %Flow3 +; GFX9-NEXT: s_or_b64 exec, exec, s[8:9] +; GFX9-NEXT: v_mul_lo_u32 v16, v13, v5 +; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v23, v13, 0 +; GFX9-NEXT: v_mov_b32_e32 v15, 0 +; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v13, v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v14, v6 +; GFX9-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v21, v13, v[14:15] +; GFX9-NEXT: v_mul_lo_u32 v9, v10, v4 +; GFX9-NEXT: v_mul_lo_u32 v11, v11, v23 +; GFX9-NEXT: v_mov_b32_e32 v4, v14 +; GFX9-NEXT: v_mov_b32_e32 v14, v15 +; GFX9-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v23, v10, v[13:14] +; GFX9-NEXT: v_add3_u32 v8, v8, v16, v9 +; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v12, v23, v[7:8] +; GFX9-NEXT: v_mov_b32_e32 v8, v14 +; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v4, v8 +; GFX9-NEXT: v_addc_co_u32_e64 v9, s[4:5], 0, 0, vcc +; GFX9-NEXT: v_mul_lo_u32 v12, v12, v21 +; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v21, v10, v[8:9] +; GFX9-NEXT: v_add3_u32 v4, v11, v7, v12 +; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v8, v6 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v9, v4, vcc +; GFX9-NEXT: v_mov_b32_e32 v7, v13 +; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v5 +; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v7, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v6, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v4, vcc +; GFX9-NEXT: v_xor_b32_e32 v5, v0, v20 +; GFX9-NEXT: v_xor_b32_e32 v0, v2, v20 +; GFX9-NEXT: v_xor_b32_e32 v4, v1, v22 +; GFX9-NEXT: v_xor_b32_e32 v1, v3, v22 +; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v20 +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v22, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v5, v20, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v22, vcc +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-O0-LABEL: v_srem_i128_vv: +; GFX9-O0: ; %bb.0: ; %_udiv-special-cases +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-O0-NEXT: ; implicit-def: $vgpr8 : SGPR spill to VGPR lane +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v2 +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v1 +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v0 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v7 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec +; GFX9-O0-NEXT: s_waitcnt vmcnt(1) +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v1 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 +; GFX9-O0-NEXT: s_mov_b32 s4, 63 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v14 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v15 +; GFX9-O0-NEXT: v_ashrrev_i64 v[12:13], s4, v[6:7] +; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v13 +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v9 +; GFX9-O0-NEXT: v_ashrrev_i64 v[6:7], s4, v[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v15 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v13 +; GFX9-O0-NEXT: v_xor_b32_e64 v1, v1, v10 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v14 +; GFX9-O0-NEXT: v_xor_b32_e64 v13, v11, v12 +; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-O0-NEXT: v_xor_b32_e64 v1, v1, v10 +; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec +; GFX9-O0-NEXT: v_xor_b32_e64 v15, v4, v12 +; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v7 +; GFX9-O0-NEXT: v_xor_b32_e64 v1, v1, v4 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v8 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_xor_b32_e64 v7, v5, v6 +; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3 +; GFX9-O0-NEXT: v_xor_b32_e64 v1, v1, v4 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_xor_b32_e64 v2, v2, v6 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v15 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v16 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v14 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v9, vcc, v9, v12 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v10, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v13, vcc, v11, v12, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v5, vcc, v5, v10, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v2 +; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v8 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v6 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v4, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v11, vcc, v5, v6, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v4, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v2 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v12 +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14 +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v2 +; GFX9-O0-NEXT: v_or_b32_e64 v3, v8, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 +; GFX9-O0-NEXT: v_or_b32_e64 v1, v5, v6 +; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 +; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 0 +; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 1 +; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[1:2], s[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v14 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 +; GFX9-O0-NEXT: v_or_b32_e64 v15, v4, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 +; GFX9-O0-NEXT: v_or_b32_e64 v9, v3, v1 +; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v15 +; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[9:10], s[6:7] +; GFX9-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9] +; GFX9-O0-NEXT: v_ffbh_u32_e64 v6, v6 +; GFX9-O0-NEXT: s_mov_b32 s9, 32 +; GFX9-O0-NEXT: v_add_u32_e64 v6, v6, s9 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v7, v7 +; GFX9-O0-NEXT: v_min_u32_e64 v6, v6, v7 +; GFX9-O0-NEXT: s_mov_b32 s8, 0 +; GFX9-O0-NEXT: ; implicit-def: $sgpr10 +; GFX9-O0-NEXT: v_mov_b32_e32 v9, s8 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v7 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v5 +; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v8, v8 +; GFX9-O0-NEXT: v_min_u32_e64 v15, v5, v8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr10 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 +; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v5 +; GFX9-O0-NEXT: s_mov_b64 s[10:11], 64 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v15 +; GFX9-O0-NEXT: s_mov_b32 s12, s10 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16 +; GFX9-O0-NEXT: s_mov_b32 s14, s11 +; GFX9-O0-NEXT: v_add_co_u32_e64 v8, s[12:13], v8, s12 +; GFX9-O0-NEXT: v_mov_b32_e32 v9, s14 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[12:13], v5, v9, s[12:13] +; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 +; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[12:13], v[11:12], s[6:7] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v10, s[12:13] +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v8 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v9, v6, v7, s[12:13] +; GFX9-O0-NEXT: ; implicit-def: $sgpr12 +; GFX9-O0-NEXT: ; implicit-def: $sgpr12 +; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v1 +; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v6, v2 +; GFX9-O0-NEXT: v_min_u32_e64 v6, v5, v6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr12 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v7 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v3 +; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v11, v4 +; GFX9-O0-NEXT: v_min_u32_e64 v15, v5, v11 +; GFX9-O0-NEXT: ; implicit-def: $sgpr9 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 +; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v15 +; GFX9-O0-NEXT: s_mov_b32 s8, s10 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16 +; GFX9-O0-NEXT: s_mov_b32 s10, s11 +; GFX9-O0-NEXT: v_add_co_u32_e64 v11, s[8:9], v11, s8 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, s10 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[8:9], v5, v12, s[8:9] +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 +; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[8:9], v[13:14], s[6:7] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[8:9] +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[8:9] +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 +; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 +; GFX9-O0-NEXT: s_mov_b32 s10, s6 +; GFX9-O0-NEXT: s_mov_b32 s11, s7 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v5, vcc, v5, v8 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v9, vcc, v6, v7, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v7, s10 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, s10 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v8, vcc, v6, v7, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v7, s11 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, s11 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v6, v7, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v9 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v8 +; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 +; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] +; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[8:9], s[8:9] +; GFX9-O0-NEXT: s_mov_b64 s[12:13], 0x7f +; GFX9-O0-NEXT: s_mov_b64 s[14:15], s[12:13] +; GFX9-O0-NEXT: v_cmp_gt_u64_e64 s[14:15], v[5:6], s[14:15] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[14:15] +; GFX9-O0-NEXT: s_mov_b64 s[14:15], s[6:7] +; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[14:15], v[8:9], s[14:15] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[14:15] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, v7, v10, s[8:9] +; GFX9-O0-NEXT: v_and_b32_e64 v7, 1, v7 +; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[8:9], v7, 1 +; GFX9-O0-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9] +; GFX9-O0-NEXT: s_mov_b64 s[4:5], -1 +; GFX9-O0-NEXT: s_xor_b64 s[4:5], s[8:9], s[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 +; GFX9-O0-NEXT: s_mov_b32 s14, s13 +; GFX9-O0-NEXT: v_xor_b32_e64 v7, v7, s14 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13 +; GFX9-O0-NEXT: v_xor_b32_e64 v5, v5, s12 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v9 +; GFX9-O0-NEXT: v_or_b32_e64 v7, v7, v10 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v8 +; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v6 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7 +; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[5:6], s[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s11 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v2, v5, s[8:9] +; GFX9-O0-NEXT: v_mov_b32_e32 v2, s10 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v1, v1, v2, s[8:9] +; GFX9-O0-NEXT: ; implicit-def: $sgpr12 +; GFX9-O0-NEXT: ; implicit-def: $sgpr12 +; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s11 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v4, v5, s[8:9] +; GFX9-O0-NEXT: v_mov_b32_e32 v4, s10 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[8:9] +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v5 +; GFX9-O0-NEXT: s_and_b64 s[6:7], s[4:5], s[6:7] +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 s[4:5], exec +; GFX9-O0-NEXT: v_writelane_b32 v0, s4, 2 +; GFX9-O0-NEXT: v_writelane_b32 v0, s5, 3 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7] +; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-O0-NEXT: s_cbranch_execz .LBB0_3 +; GFX9-O0-NEXT: s_branch .LBB0_8 +; GFX9-O0-NEXT: .LBB0_1: ; %Flow +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_readlane_b32 s4, v0, 4 +; GFX9-O0-NEXT: v_readlane_b32 s5, v0, 5 +; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-O0-NEXT: ; %bb.2: ; %Flow +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_waitcnt vmcnt(6) +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_branch .LBB0_5 +; GFX9-O0-NEXT: .LBB0_3: ; %Flow2 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_readlane_b32 s4, v4, 2 +; GFX9-O0-NEXT: v_readlane_b32 s5, v4, 3 +; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_branch .LBB0_9 +; GFX9-O0-NEXT: .LBB0_4: ; %udiv-loop-exit +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b32 s4, 1 +; GFX9-O0-NEXT: s_waitcnt vmcnt(2) +; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[0:1] +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_lshlrev_b64 v[9:10], s4, v[9:10] +; GFX9-O0-NEXT: s_mov_b32 s4, 63 +; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1] +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v8 +; GFX9-O0-NEXT: v_or3_b32 v4, v4, v11, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v9 +; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v7 +; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v4 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v6 +; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5 +; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v4 +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_branch .LBB0_3 +; GFX9-O0-NEXT: .LBB0_5: ; %Flow1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_readlane_b32 s4, v8, 6 +; GFX9-O0-NEXT: v_readlane_b32 s5, v8, 7 +; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_branch .LBB0_4 +; GFX9-O0-NEXT: .LBB0_6: ; %udiv-do-while +; GFX9-O0-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_readlane_b32 s6, v16, 8 +; GFX9-O0-NEXT: v_readlane_b32 s7, v16, 9 +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b32 s4, 63 +; GFX9-O0-NEXT: s_waitcnt vmcnt(16) +; GFX9-O0-NEXT: v_lshrrev_b64 v[29:30], s4, v[2:3] +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v30 +; GFX9-O0-NEXT: s_mov_b32 s5, 1 +; GFX9-O0-NEXT: v_lshlrev_b64 v[23:24], s5, v[23:24] +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v24 +; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v29 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v23 +; GFX9-O0-NEXT: v_or_b32_e64 v23, v5, v10 +; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v24, v4 +; GFX9-O0-NEXT: v_lshlrev_b64 v[29:30], s5, v[2:3] +; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], s4, v[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v30 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v29 +; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec +; GFX9-O0-NEXT: v_or_b32_e64 v4, v3, v4 +; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v2 +; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s5, v[0:1] +; GFX9-O0-NEXT: v_lshlrev_b64 v[29:30], s5, v[6:7] +; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1] +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v30 +; GFX9-O0-NEXT: s_waitcnt vmcnt(10) +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v28 +; GFX9-O0-NEXT: v_or3_b32 v6, v6, v7, v10 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v29 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v27 +; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v7 +; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3 +; GFX9-O0-NEXT: s_waitcnt vmcnt(8) +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v26 +; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v25 +; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v4 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v23 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v24 +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v14 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v15 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v13, vcc, v13, v6 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v12, vcc, v12, v10, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v11, vcc, v11, v4, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v7, v5, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v7 +; GFX9-O0-NEXT: v_ashrrev_i64 v[13:14], s4, v[11:12] +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v14 +; GFX9-O0-NEXT: s_mov_b64 s[4:5], 1 +; GFX9-O0-NEXT: s_mov_b32 s8, s5 +; GFX9-O0-NEXT: v_and_b32_e64 v12, v7, s8 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v13 +; GFX9-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5 +; GFX9-O0-NEXT: v_and_b32_e64 v14, v11, s4 +; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v23, v22 +; GFX9-O0-NEXT: v_and_b32_e64 v23, v7, v23 +; GFX9-O0-NEXT: v_and_b32_e64 v21, v11, v21 +; GFX9-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v22, v23 +; GFX9-O0-NEXT: v_mov_b32_e32 v23, v20 +; GFX9-O0-NEXT: v_and_b32_e64 v7, v7, v23 +; GFX9-O0-NEXT: v_and_b32_e64 v23, v11, v19 +; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v24, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v23 +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v24 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v21 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v22 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v6, vcc, v6, v20 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v10, vcc, v10, v19, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v4, vcc, v4, v11, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v5, v7, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v10 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v8 +; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 killed $vgpr8_vgpr9 killed $exec +; GFX9-O0-NEXT: s_mov_b64 s[8:9], -1 +; GFX9-O0-NEXT: s_mov_b32 s5, s8 +; GFX9-O0-NEXT: s_mov_b32 s4, s9 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v18 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, s5 +; GFX9-O0-NEXT: v_add_co_u32_e32 v20, vcc, v11, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s4 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v9, vcc, v9, v11, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s5 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v17, vcc, v10, v11, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v10, s4 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v8, vcc, v8, v10, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr20 killed $vgpr20 def $vgpr20_vgpr21 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v21, v9 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v18 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v20 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v21 +; GFX9-O0-NEXT: v_mov_b32_e32 v22, v18 +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v21 +; GFX9-O0-NEXT: v_or_b32_e64 v19, v19, v22 +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, v20 +; GFX9-O0-NEXT: v_or_b32_e64 v17, v17, v18 +; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v19 +; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[17:18], v[12:13] +; GFX9-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, v2 +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, v0 +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v15 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, v14 +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, v12 +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5] +; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 4 +; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 5 +; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5] +; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 8 +; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 9 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX9-O0-NEXT: s_cbranch_execnz .LBB0_6 +; GFX9-O0-NEXT: s_branch .LBB0_1 +; GFX9-O0-NEXT: .LBB0_7: ; %udiv-preheader +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_waitcnt vmcnt(9) +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], v4, v[21:22] +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7 +; GFX9-O0-NEXT: s_mov_b32 s6, 64 +; GFX9-O0-NEXT: v_sub_u32_e64 v12, s6, v4 +; GFX9-O0-NEXT: v_lshlrev_b64 v[23:24], v12, v[19:20] +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v24 +; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v12 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v23 +; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v7 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v7 +; GFX9-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v4, s6 +; GFX9-O0-NEXT: v_sub_u32_e64 v5, v4, s6 +; GFX9-O0-NEXT: v_lshrrev_b64 v[23:24], v5, v[19:20] +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v24 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v12, s[4:5] +; GFX9-O0-NEXT: s_mov_b32 s6, 0 +; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v4, s6 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v22 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v12, s[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v23 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v21 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[6:7] +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 +; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[19:20] +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v5 +; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 +; GFX9-O0-NEXT: s_mov_b32 s8, s7 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, s8 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v12, v12, v15, s[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4 +; GFX9-O0-NEXT: s_mov_b32 s8, s6 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[4:5] +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v14 +; GFX9-O0-NEXT: s_mov_b64 s[8:9], -1 +; GFX9-O0-NEXT: s_mov_b32 s5, s8 +; GFX9-O0-NEXT: s_mov_b32 s4, s9 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v18 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, s5 +; GFX9-O0-NEXT: v_add_co_u32_e32 v12, vcc, v12, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, s4 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v17, vcc, v15, v17, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v15, s5 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v14, vcc, v14, v15, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v15, s4 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v13, vcc, v13, v15, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v13 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v17 +; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v15, s9 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, s7 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, s6 +; GFX9-O0-NEXT: v_writelane_b32 v16, s4, 8 +; GFX9-O0-NEXT: v_writelane_b32 v16, s5, 9 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_branch .LBB0_6 +; GFX9-O0-NEXT: .LBB0_8: ; %udiv-bb1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 s[6:7], 1 +; GFX9-O0-NEXT: s_mov_b32 s5, s6 +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v1 +; GFX9-O0-NEXT: s_mov_b32 s4, s7 +; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 +; GFX9-O0-NEXT: s_mov_b32 s8, s6 +; GFX9-O0-NEXT: s_mov_b32 s9, s7 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s5 +; GFX9-O0-NEXT: v_add_co_u32_e32 v9, vcc, v4, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s4 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v5, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s9 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v5, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v2 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b32 s4, 0x7f +; GFX9-O0-NEXT: v_sub_u32_e64 v3, s4, v4 +; GFX9-O0-NEXT: v_lshlrev_b64 v[5:6], v3, v[11:12] +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v6 +; GFX9-O0-NEXT: s_mov_b32 s4, 64 +; GFX9-O0-NEXT: v_sub_u32_e64 v14, s4, v3 +; GFX9-O0-NEXT: v_lshrrev_b64 v[14:15], v14, v[7:8] +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v15 +; GFX9-O0-NEXT: v_or_b32_e64 v13, v13, v16 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v14 +; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v6 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v6 +; GFX9-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v3, s4 +; GFX9-O0-NEXT: s_mov_b32 s10, 63 +; GFX9-O0-NEXT: v_sub_u32_e64 v4, s10, v4 +; GFX9-O0-NEXT: v_lshlrev_b64 v[13:14], v4, v[7:8] +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v15, s[4:5] +; GFX9-O0-NEXT: s_mov_b32 s10, 0 +; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[10:11], v3, s10 +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v15, s[10:11] +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v13 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[10:11] +; GFX9-O0-NEXT: ; implicit-def: $sgpr10 +; GFX9-O0-NEXT: ; implicit-def: $sgpr10 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v4 +; GFX9-O0-NEXT: v_lshlrev_b64 v[7:8], v3, v[7:8] +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, s9 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[4:5] +; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr7_vgpr8 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, v4, v7, s[4:5] +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v3 +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v10 +; GFX9-O0-NEXT: v_or_b32_e64 v3, v3, v4 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v9 +; GFX9-O0-NEXT: v_or_b32_e64 v1, v1, v2 +; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[4:5], v[1:2], s[6:7] +; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v3, s8 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, s9 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, s6 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, s7 +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 s[6:7], exec +; GFX9-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5] +; GFX9-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7] +; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 6 +; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 7 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-O0-NEXT: s_cbranch_execz .LBB0_5 +; GFX9-O0-NEXT: s_branch .LBB0_7 +; GFX9-O0-NEXT: .LBB0_9: ; %udiv-end +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b32 s4, 32 +; GFX9-O0-NEXT: s_waitcnt vmcnt(2) +; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[5:6] +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v17 +; GFX9-O0-NEXT: v_mul_lo_u32 v3, v1, v0 +; GFX9-O0-NEXT: v_lshrrev_b64 v[17:18], s4, v[17:18] +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v17 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mul_lo_u32 v2, v5, v2 +; GFX9-O0-NEXT: v_mad_u64_u32 v[17:18], s[6:7], v5, v0, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v18 +; GFX9-O0-NEXT: v_add3_u32 v2, v0, v2, v3 +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v0 +; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[2:3] +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v3 +; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 killed $vgpr17_vgpr18 killed $exec +; GFX9-O0-NEXT: s_mov_b32 s5, 0 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v18 +; GFX9-O0-NEXT: v_or_b32_e64 v0, v0, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v17 +; GFX9-O0-NEXT: v_or_b32_e64 v17, v2, v3 +; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v0 +; GFX9-O0-NEXT: v_lshrrev_b64 v[2:3], s4, v[19:20] +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v11 +; GFX9-O0-NEXT: v_mul_lo_u32 v3, v2, v6 +; GFX9-O0-NEXT: v_lshrrev_b64 v[11:12], s4, v[11:12] +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 killed $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v19 +; GFX9-O0-NEXT: v_mul_lo_u32 v11, v11, v0 +; GFX9-O0-NEXT: v_mad_u64_u32 v[19:20], s[6:7], v2, v0, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v20 +; GFX9-O0-NEXT: v_add3_u32 v2, v2, v3, v11 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s6 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 +; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[2:3] +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v3 +; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 killed $vgpr19_vgpr20 killed $exec +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v20 +; GFX9-O0-NEXT: v_or_b32_e64 v11, v11, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v19 +; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v18 +; GFX9-O0-NEXT: v_add_co_u32_e64 v17, s[6:7], v11, v12 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v2, s[6:7], v2, v3, s[6:7] +; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v2 +; GFX9-O0-NEXT: v_mad_u64_u32 v[19:20], s[6:7], v6, v1, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v19 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v20 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, s6 +; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v12 +; GFX9-O0-NEXT: v_lshlrev_b64 v[19:20], s4, v[19:20] +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v20 +; GFX9-O0-NEXT: v_or_b32_e64 v11, v11, v12 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v19 +; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 +; GFX9-O0-NEXT: v_mad_u64_u32 v[19:20], s[6:7], v6, v5, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v19 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v20 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: v_mov_b32_e32 v21, s6 +; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v21 +; GFX9-O0-NEXT: v_lshlrev_b64 v[19:20], s4, v[19:20] +; GFX9-O0-NEXT: v_mov_b32_e32 v21, v20 +; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v21 +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 killed $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v19 +; GFX9-O0-NEXT: v_or_b32_e64 v23, v11, v12 +; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v24, v6 +; GFX9-O0-NEXT: v_mad_u64_u32 v[11:12], s[6:7], v0, v5, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v21, v12 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v22, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v23 +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v21 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v24 +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v22 +; GFX9-O0-NEXT: v_add_co_u32_e64 v5, s[6:7], v5, v20 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v19, s[6:7], v6, v19, s[6:7] +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v19 +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v6 +; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0xffffffff +; GFX9-O0-NEXT: s_mov_b32 s8, s7 +; GFX9-O0-NEXT: v_and_b32_e64 v19, v19, s8 +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v5 +; GFX9-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 killed $sgpr6_sgpr7 +; GFX9-O0-NEXT: v_and_b32_e64 v21, v20, s6 +; GFX9-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v22, v19 +; GFX9-O0-NEXT: v_mad_u64_u32 v[19:20], s[6:7], v0, v1, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v23, v19 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v24, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v24 +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v20 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, s6 +; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v1 +; GFX9-O0-NEXT: v_lshlrev_b64 v[19:20], s4, v[19:20] +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v20 +; GFX9-O0-NEXT: v_or_b32_e64 v0, v0, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v23 +; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 killed $vgpr19_vgpr20 killed $exec +; GFX9-O0-NEXT: v_or_b32_e64 v23, v1, v19 +; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v24, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v23 +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v21 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v24 +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v22 +; GFX9-O0-NEXT: v_add_co_u32_e64 v0, s[6:7], v0, v20 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v19, s[6:7], v1, v19, s[6:7] +; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v19 +; GFX9-O0-NEXT: v_lshrrev_b64 v[21:22], s4, v[0:1] +; GFX9-O0-NEXT: v_lshrrev_b64 v[5:6], s4, v[5:6] +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v21 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v22 +; GFX9-O0-NEXT: v_add_co_u32_e64 v19, s[6:7], v19, v20 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v5, v6, s[6:7] +; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v19 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v20 +; GFX9-O0-NEXT: v_add_co_u32_e64 v19, s[6:7], v5, v6 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v2, s[6:7], v2, v3, s[6:7] +; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v19 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v20 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v18 +; GFX9-O0-NEXT: v_add_co_u32_e64 v2, s[6:7], v2, v6 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v3, v5, s[6:7] +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-O0-NEXT: v_lshlrev_b64 v[0:1], s4, v[0:1] +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v1 +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 killed $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 +; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v11 +; GFX9-O0-NEXT: v_or_b32_e64 v0, v0, v1 +; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v15 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v16 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v14 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v12 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v11, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v5, vcc, v5, v6, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v2, vcc, v1, v2, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3 +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v10 +; GFX9-O0-NEXT: v_xor_b32_e64 v3, v3, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 +; GFX9-O0-NEXT: v_xor_b32_e64 v9, v6, v5 +; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v8 +; GFX9-O0-NEXT: v_xor_b32_e64 v3, v3, v6 +; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v7 +; GFX9-O0-NEXT: v_xor_b32_e64 v0, v0, v8 +; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v10 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v7, vcc, v7, v8 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v6, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v5, vcc, v3, v5, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v2, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v1 +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5 +; GFX9-O0-NEXT: v_lshrrev_b64 v[7:8], s4, v[7:8] +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 +; GFX9-O0-NEXT: v_lshrrev_b64 v[5:6], s4, v[5:6] +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-O0-NEXT: ; kill: killed $vgpr4 +; GFX9-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_nop 0 +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: s_setpc_b64 s[30:31] + %div = srem i128 %lhs, %rhs + ret i128 %div +} + +define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) { +; GFX9-LABEL: v_urem_i128_vv: +; GFX9: ; %bb.0: ; %_udiv-special-cases +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_or_b32_e32 v9, v5, v7 +; GFX9-NEXT: v_or_b32_e32 v8, v4, v6 +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] +; GFX9-NEXT: v_or_b32_e32 v9, v1, v3 +; GFX9-NEXT: v_or_b32_e32 v8, v0, v2 +; GFX9-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[8:9] +; GFX9-NEXT: v_ffbh_u32_e32 v8, v6 +; GFX9-NEXT: v_add_u32_e32 v8, 32, v8 +; GFX9-NEXT: v_ffbh_u32_e32 v9, v7 +; GFX9-NEXT: v_min_u32_e32 v8, v8, v9 +; GFX9-NEXT: v_ffbh_u32_e32 v9, v4 +; GFX9-NEXT: v_add_u32_e32 v9, 32, v9 +; GFX9-NEXT: v_ffbh_u32_e32 v10, v5 +; GFX9-NEXT: v_min_u32_e32 v9, v9, v10 +; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, 64, v9 +; GFX9-NEXT: v_addc_co_u32_e64 v10, s[6:7], 0, 0, vcc +; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7] +; GFX9-NEXT: v_ffbh_u32_e32 v11, v3 +; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc +; GFX9-NEXT: v_ffbh_u32_e32 v9, v2 +; GFX9-NEXT: v_add_u32_e32 v9, 32, v9 +; GFX9-NEXT: v_min_u32_e32 v9, v9, v11 +; GFX9-NEXT: v_ffbh_u32_e32 v11, v0 +; GFX9-NEXT: v_add_u32_e32 v11, 32, v11 +; GFX9-NEXT: v_ffbh_u32_e32 v12, v1 +; GFX9-NEXT: v_min_u32_e32 v11, v11, v12 +; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc +; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, 64, v11 +; GFX9-NEXT: v_addc_co_u32_e64 v12, s[6:7], 0, 0, vcc +; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] +; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f +; GFX9-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, 0, vcc +; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v8, v9 +; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, v10, v12, vcc +; GFX9-NEXT: v_mov_b32_e32 v11, 0 +; GFX9-NEXT: v_subbrev_co_u32_e32 v10, vcc, 0, v11, vcc +; GFX9-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v11, vcc +; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9] +; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc +; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11] +; GFX9-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11] +; GFX9-NEXT: v_cndmask_b32_e32 v12, v13, v12, vcc +; GFX9-NEXT: v_and_b32_e32 v12, 1, v12 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v12 +; GFX9-NEXT: v_xor_b32_e32 v12, 0x7f, v8 +; GFX9-NEXT: v_or_b32_e32 v13, v9, v11 +; GFX9-NEXT: v_or_b32_e32 v12, v12, v10 +; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], vcc +; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13] +; GFX9-NEXT: s_xor_b64 s[6:7], s[4:5], -1 +; GFX9-NEXT: v_cndmask_b32_e64 v15, v3, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v14, v2, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v13, v1, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v12, v0, 0, s[4:5] +; GFX9-NEXT: s_and_b64 s[4:5], s[6:7], vcc +; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] +; GFX9-NEXT: s_cbranch_execz .LBB1_6 +; GFX9-NEXT: ; %bb.1: ; %udiv-bb1 +; GFX9-NEXT: v_add_co_u32_e32 v22, vcc, 1, v8 +; GFX9-NEXT: v_addc_co_u32_e32 v23, vcc, 0, v9, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, 0, v10, vcc +; GFX9-NEXT: v_sub_u32_e32 v15, 0x7f, v8 +; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v11, vcc +; GFX9-NEXT: v_sub_u32_e32 v13, 64, v15 +; GFX9-NEXT: v_or_b32_e32 v10, v23, v25 +; GFX9-NEXT: v_or_b32_e32 v9, v22, v24 +; GFX9-NEXT: v_lshlrev_b64 v[11:12], v15, v[2:3] +; GFX9-NEXT: v_lshrrev_b64 v[13:14], v13, v[0:1] +; GFX9-NEXT: v_sub_u32_e32 v8, 63, v8 +; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[9:10] +; GFX9-NEXT: v_lshlrev_b64 v[8:9], v8, v[0:1] +; GFX9-NEXT: v_or_b32_e32 v10, v12, v14 +; GFX9-NEXT: v_or_b32_e32 v11, v11, v13 +; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v15 +; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v10, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v11, s[4:5] +; GFX9-NEXT: v_lshlrev_b64 v[10:11], v15, v[0:1] +; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v15 +; GFX9-NEXT: v_mov_b32_e32 v12, 0 +; GFX9-NEXT: v_mov_b32_e32 v14, 0 +; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v3, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v2, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, v11, s[4:5] +; GFX9-NEXT: v_mov_b32_e32 v13, 0 +; GFX9-NEXT: v_mov_b32_e32 v15, 0 +; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, v10, s[4:5] +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_xor_b64 s[6:7], exec, s[4:5] +; GFX9-NEXT: s_cbranch_execz .LBB1_5 +; GFX9-NEXT: ; %bb.2: ; %udiv-preheader +; GFX9-NEXT: v_sub_u32_e32 v14, 64, v22 +; GFX9-NEXT: v_lshrrev_b64 v[12:13], v22, v[0:1] +; GFX9-NEXT: v_lshlrev_b64 v[14:15], v14, v[2:3] +; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v22 +; GFX9-NEXT: v_or_b32_e32 v14, v12, v14 +; GFX9-NEXT: v_subrev_u32_e32 v12, 64, v22 +; GFX9-NEXT: v_or_b32_e32 v15, v13, v15 +; GFX9-NEXT: v_lshrrev_b64 v[12:13], v12, v[2:3] +; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v22 +; GFX9-NEXT: v_cndmask_b32_e32 v13, v13, v15, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v17, v13, v1, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v14, v12, v14, vcc +; GFX9-NEXT: v_lshrrev_b64 v[12:13], v22, v[2:3] +; GFX9-NEXT: v_cndmask_b32_e64 v16, v14, v0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v19, 0, v13, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v18, 0, v12, vcc +; GFX9-NEXT: v_add_co_u32_e32 v26, vcc, -1, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, -1, v5, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v28, vcc, -1, v6, vcc +; GFX9-NEXT: v_mov_b32_e32 v20, 0 +; GFX9-NEXT: v_mov_b32_e32 v14, 0 +; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v7, vcc +; GFX9-NEXT: s_mov_b64 s[4:5], 0 +; GFX9-NEXT: v_mov_b32_e32 v21, 0 +; GFX9-NEXT: v_mov_b32_e32 v15, 0 +; GFX9-NEXT: v_mov_b32_e32 v13, 0 +; GFX9-NEXT: .LBB1_3: ; %udiv-do-while +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_lshrrev_b32_e32 v12, 31, v11 +; GFX9-NEXT: v_lshlrev_b64 v[10:11], 1, v[10:11] +; GFX9-NEXT: v_lshlrev_b64 v[18:19], 1, v[18:19] +; GFX9-NEXT: v_or_b32_e32 v10, v20, v10 +; GFX9-NEXT: v_lshrrev_b32_e32 v20, 31, v17 +; GFX9-NEXT: v_lshlrev_b64 v[16:17], 1, v[16:17] +; GFX9-NEXT: v_or_b32_e32 v18, v18, v20 +; GFX9-NEXT: v_lshrrev_b32_e32 v20, 31, v9 +; GFX9-NEXT: v_or_b32_e32 v16, v16, v20 +; GFX9-NEXT: v_sub_co_u32_e32 v20, vcc, v26, v16 +; GFX9-NEXT: v_subb_co_u32_e32 v20, vcc, v27, v17, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v20, vcc, v28, v18, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v20, vcc, v29, v19, vcc +; GFX9-NEXT: v_ashrrev_i32_e32 v30, 31, v20 +; GFX9-NEXT: v_and_b32_e32 v20, v30, v4 +; GFX9-NEXT: v_sub_co_u32_e32 v16, vcc, v16, v20 +; GFX9-NEXT: v_and_b32_e32 v20, v30, v5 +; GFX9-NEXT: v_subb_co_u32_e32 v17, vcc, v17, v20, vcc +; GFX9-NEXT: v_and_b32_e32 v20, v30, v6 +; GFX9-NEXT: v_subb_co_u32_e32 v18, vcc, v18, v20, vcc +; GFX9-NEXT: v_and_b32_e32 v20, v30, v7 +; GFX9-NEXT: v_subb_co_u32_e32 v19, vcc, v19, v20, vcc +; GFX9-NEXT: v_add_co_u32_e32 v22, vcc, -1, v22 +; GFX9-NEXT: v_addc_co_u32_e32 v23, vcc, -1, v23, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, -1, v24, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v25, vcc +; GFX9-NEXT: v_or_b32_e32 v11, v21, v11 +; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9] +; GFX9-NEXT: v_or_b32_e32 v20, v22, v24 +; GFX9-NEXT: v_or_b32_e32 v21, v23, v25 +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[20:21] +; GFX9-NEXT: v_or3_b32 v8, v8, v12, v14 +; GFX9-NEXT: v_and_b32_e32 v12, 1, v30 +; GFX9-NEXT: v_mov_b32_e32 v21, v13 +; GFX9-NEXT: v_or3_b32 v9, v9, 0, v15 +; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX9-NEXT: v_mov_b32_e32 v20, v12 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX9-NEXT: s_cbranch_execnz .LBB1_3 +; GFX9-NEXT: ; %bb.4: ; %Flow +; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-NEXT: .LBB1_5: ; %Flow2 +; GFX9-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX9-NEXT: v_lshlrev_b64 v[16:17], 1, v[10:11] +; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9] +; GFX9-NEXT: v_lshrrev_b32_e32 v10, 31, v11 +; GFX9-NEXT: v_or3_b32 v15, v9, 0, v15 +; GFX9-NEXT: v_or3_b32 v14, v8, v10, v14 +; GFX9-NEXT: v_or_b32_e32 v13, v13, v17 +; GFX9-NEXT: v_or_b32_e32 v12, v12, v16 +; GFX9-NEXT: .LBB1_6: ; %Flow3 +; GFX9-NEXT: s_or_b64 exec, exec, s[8:9] +; GFX9-NEXT: v_mul_lo_u32 v19, v12, v7 +; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v4, v12, 0 +; GFX9-NEXT: v_mov_b32_e32 v17, 0 +; GFX9-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v12, v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v16, v8 +; GFX9-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v5, v12, v[16:17] +; GFX9-NEXT: v_mul_lo_u32 v18, v13, v6 +; GFX9-NEXT: v_mul_lo_u32 v16, v15, v4 +; GFX9-NEXT: v_mov_b32_e32 v6, v12 +; GFX9-NEXT: v_mov_b32_e32 v12, v17 +; GFX9-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v4, v13, v[11:12] +; GFX9-NEXT: v_add3_u32 v10, v10, v19, v18 +; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v14, v4, v[9:10] +; GFX9-NEXT: v_mov_b32_e32 v4, v12 +; GFX9-NEXT: v_mul_lo_u32 v10, v14, v5 +; GFX9-NEXT: v_add_co_u32_e32 v14, vcc, v6, v4 +; GFX9-NEXT: v_addc_co_u32_e64 v15, s[4:5], 0, 0, vcc +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v5, v13, v[14:15] +; GFX9-NEXT: v_add3_u32 v6, v16, v9, v10 +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v8 +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v6, vcc +; GFX9-NEXT: v_mov_b32_e32 v6, v11 +; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v7 +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v6, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v4, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v5, vcc +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-O0-LABEL: v_urem_i128_vv: +; GFX9-O0: ; %bb.0: ; %_udiv-special-cases +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-O0-NEXT: ; implicit-def: $vgpr8 : SGPR spill to VGPR lane +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v6 +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v1 +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v0 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v2 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v3 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v7 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14 +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v12 +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v12 +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14 +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v2 +; GFX9-O0-NEXT: v_or_b32_e64 v3, v8, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 +; GFX9-O0-NEXT: v_or_b32_e64 v1, v5, v6 +; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 +; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 0 +; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 1 +; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[1:2], s[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v14 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 +; GFX9-O0-NEXT: v_or_b32_e64 v15, v4, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 +; GFX9-O0-NEXT: v_or_b32_e64 v9, v3, v1 +; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v15 +; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[9:10], s[6:7] +; GFX9-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9] +; GFX9-O0-NEXT: v_ffbh_u32_e64 v6, v6 +; GFX9-O0-NEXT: s_mov_b32 s9, 32 +; GFX9-O0-NEXT: v_add_u32_e64 v6, v6, s9 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v7, v7 +; GFX9-O0-NEXT: v_min_u32_e64 v6, v6, v7 +; GFX9-O0-NEXT: s_mov_b32 s8, 0 +; GFX9-O0-NEXT: ; implicit-def: $sgpr10 +; GFX9-O0-NEXT: v_mov_b32_e32 v9, s8 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v7 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v5 +; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v8, v8 +; GFX9-O0-NEXT: v_min_u32_e64 v15, v5, v8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr10 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 +; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v5 +; GFX9-O0-NEXT: s_mov_b64 s[10:11], 64 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v15 +; GFX9-O0-NEXT: s_mov_b32 s12, s10 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16 +; GFX9-O0-NEXT: s_mov_b32 s14, s11 +; GFX9-O0-NEXT: v_add_co_u32_e64 v8, s[12:13], v8, s12 +; GFX9-O0-NEXT: v_mov_b32_e32 v9, s14 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[12:13], v5, v9, s[12:13] +; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 +; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[6:7] +; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[12:13], v[11:12], s[12:13] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v10, s[12:13] +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v8 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v9, v6, v7, s[12:13] +; GFX9-O0-NEXT: ; implicit-def: $sgpr12 +; GFX9-O0-NEXT: ; implicit-def: $sgpr12 +; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v1 +; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v6, v2 +; GFX9-O0-NEXT: v_min_u32_e64 v6, v5, v6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr12 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v7 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v3 +; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v11, v4 +; GFX9-O0-NEXT: v_min_u32_e64 v15, v5, v11 +; GFX9-O0-NEXT: ; implicit-def: $sgpr9 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 +; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v15 +; GFX9-O0-NEXT: s_mov_b32 s8, s10 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16 +; GFX9-O0-NEXT: s_mov_b32 s10, s11 +; GFX9-O0-NEXT: v_add_co_u32_e64 v11, s[8:9], v11, s8 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, s10 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[8:9], v5, v12, s[8:9] +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 +; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] +; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[8:9], v[13:14], s[8:9] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[8:9] +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[8:9] +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 +; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 +; GFX9-O0-NEXT: s_mov_b32 s10, s6 +; GFX9-O0-NEXT: s_mov_b32 s11, s7 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v5, vcc, v5, v8 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v9, vcc, v6, v7, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v7, s10 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, s10 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v8, vcc, v6, v7, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v7, s11 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, s11 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v6, v7, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v9 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v8 +; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 +; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] +; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[8:9], s[8:9] +; GFX9-O0-NEXT: s_mov_b64 s[12:13], 0x7f +; GFX9-O0-NEXT: s_mov_b64 s[14:15], s[12:13] +; GFX9-O0-NEXT: v_cmp_gt_u64_e64 s[14:15], v[5:6], s[14:15] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[14:15] +; GFX9-O0-NEXT: s_mov_b64 s[14:15], s[6:7] +; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[14:15], v[8:9], s[14:15] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[14:15] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, v7, v10, s[8:9] +; GFX9-O0-NEXT: v_and_b32_e64 v7, 1, v7 +; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[8:9], v7, 1 +; GFX9-O0-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9] +; GFX9-O0-NEXT: s_mov_b64 s[4:5], -1 +; GFX9-O0-NEXT: s_xor_b64 s[4:5], s[8:9], s[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 +; GFX9-O0-NEXT: s_mov_b32 s14, s13 +; GFX9-O0-NEXT: v_xor_b32_e64 v7, v7, s14 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13 +; GFX9-O0-NEXT: v_xor_b32_e64 v5, v5, s12 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v9 +; GFX9-O0-NEXT: v_or_b32_e64 v7, v7, v10 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v8 +; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v6 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7 +; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[5:6], s[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s11 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v2, v5, s[8:9] +; GFX9-O0-NEXT: v_mov_b32_e32 v2, s10 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v1, v1, v2, s[8:9] +; GFX9-O0-NEXT: ; implicit-def: $sgpr12 +; GFX9-O0-NEXT: ; implicit-def: $sgpr12 +; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s11 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v4, v5, s[8:9] +; GFX9-O0-NEXT: v_mov_b32_e32 v4, s10 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[8:9] +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v5 +; GFX9-O0-NEXT: s_and_b64 s[6:7], s[4:5], s[6:7] +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 s[4:5], exec +; GFX9-O0-NEXT: v_writelane_b32 v0, s4, 2 +; GFX9-O0-NEXT: v_writelane_b32 v0, s5, 3 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7] +; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-O0-NEXT: s_cbranch_execz .LBB1_3 +; GFX9-O0-NEXT: s_branch .LBB1_8 +; GFX9-O0-NEXT: .LBB1_1: ; %Flow +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_readlane_b32 s4, v0, 4 +; GFX9-O0-NEXT: v_readlane_b32 s5, v0, 5 +; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-O0-NEXT: ; %bb.2: ; %Flow +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_waitcnt vmcnt(6) +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_branch .LBB1_5 +; GFX9-O0-NEXT: .LBB1_3: ; %Flow2 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_readlane_b32 s4, v4, 2 +; GFX9-O0-NEXT: v_readlane_b32 s5, v4, 3 +; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_branch .LBB1_9 +; GFX9-O0-NEXT: .LBB1_4: ; %udiv-loop-exit +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b32 s4, 1 +; GFX9-O0-NEXT: s_waitcnt vmcnt(2) +; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[0:1] +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_lshlrev_b64 v[9:10], s4, v[9:10] +; GFX9-O0-NEXT: s_mov_b32 s4, 63 +; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1] +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v8 +; GFX9-O0-NEXT: v_or3_b32 v4, v4, v11, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v9 +; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v7 +; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v4 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v6 +; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5 +; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v4 +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_branch .LBB1_3 +; GFX9-O0-NEXT: .LBB1_5: ; %Flow1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_readlane_b32 s4, v8, 6 +; GFX9-O0-NEXT: v_readlane_b32 s5, v8, 7 +; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_branch .LBB1_4 +; GFX9-O0-NEXT: .LBB1_6: ; %udiv-do-while +; GFX9-O0-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_readlane_b32 s6, v16, 8 +; GFX9-O0-NEXT: v_readlane_b32 s7, v16, 9 +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b32 s4, 63 +; GFX9-O0-NEXT: s_waitcnt vmcnt(16) +; GFX9-O0-NEXT: v_lshrrev_b64 v[29:30], s4, v[2:3] +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v30 +; GFX9-O0-NEXT: s_mov_b32 s5, 1 +; GFX9-O0-NEXT: v_lshlrev_b64 v[23:24], s5, v[23:24] +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v24 +; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v29 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v23 +; GFX9-O0-NEXT: v_or_b32_e64 v23, v5, v10 +; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v24, v4 +; GFX9-O0-NEXT: v_lshlrev_b64 v[29:30], s5, v[2:3] +; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], s4, v[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v30 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v29 +; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec +; GFX9-O0-NEXT: v_or_b32_e64 v4, v3, v4 +; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v2 +; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s5, v[0:1] +; GFX9-O0-NEXT: v_lshlrev_b64 v[29:30], s5, v[6:7] +; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1] +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v30 +; GFX9-O0-NEXT: s_waitcnt vmcnt(10) +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v28 +; GFX9-O0-NEXT: v_or3_b32 v6, v6, v7, v10 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v29 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v27 +; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v7 +; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3 +; GFX9-O0-NEXT: s_waitcnt vmcnt(8) +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v26 +; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v25 +; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v4 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v23 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v24 +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v14 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v15 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v13, vcc, v13, v6 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v12, vcc, v12, v10, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v11, vcc, v11, v4, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v7, v5, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v7 +; GFX9-O0-NEXT: v_ashrrev_i64 v[13:14], s4, v[11:12] +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v14 +; GFX9-O0-NEXT: s_mov_b64 s[4:5], 1 +; GFX9-O0-NEXT: s_mov_b32 s8, s5 +; GFX9-O0-NEXT: v_and_b32_e64 v12, v7, s8 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v13 +; GFX9-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5 +; GFX9-O0-NEXT: v_and_b32_e64 v14, v11, s4 +; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v23, v22 +; GFX9-O0-NEXT: v_and_b32_e64 v23, v7, v23 +; GFX9-O0-NEXT: v_and_b32_e64 v21, v11, v21 +; GFX9-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v22, v23 +; GFX9-O0-NEXT: v_mov_b32_e32 v23, v20 +; GFX9-O0-NEXT: v_and_b32_e64 v7, v7, v23 +; GFX9-O0-NEXT: v_and_b32_e64 v23, v11, v19 +; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v24, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v23 +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v24 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v21 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v22 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v6, vcc, v6, v20 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v10, vcc, v10, v19, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v4, vcc, v4, v11, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v5, v7, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v10 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v8 +; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 killed $vgpr8_vgpr9 killed $exec +; GFX9-O0-NEXT: s_mov_b64 s[8:9], -1 +; GFX9-O0-NEXT: s_mov_b32 s5, s8 +; GFX9-O0-NEXT: s_mov_b32 s4, s9 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v18 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, s5 +; GFX9-O0-NEXT: v_add_co_u32_e32 v20, vcc, v11, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s4 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v9, vcc, v9, v11, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s5 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v17, vcc, v10, v11, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v10, s4 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v8, vcc, v8, v10, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr20 killed $vgpr20 def $vgpr20_vgpr21 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v21, v9 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v18 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v20 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v21 +; GFX9-O0-NEXT: v_mov_b32_e32 v22, v18 +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v21 +; GFX9-O0-NEXT: v_or_b32_e64 v19, v19, v22 +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, v20 +; GFX9-O0-NEXT: v_or_b32_e64 v17, v17, v18 +; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v19 +; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[17:18], v[12:13] +; GFX9-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, v2 +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, v0 +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v15 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, v14 +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, v12 +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5] +; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 4 +; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 5 +; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5] +; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 8 +; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 9 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX9-O0-NEXT: s_cbranch_execnz .LBB1_6 +; GFX9-O0-NEXT: s_branch .LBB1_1 +; GFX9-O0-NEXT: .LBB1_7: ; %udiv-preheader +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_waitcnt vmcnt(9) +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], v4, v[21:22] +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7 +; GFX9-O0-NEXT: s_mov_b32 s6, 64 +; GFX9-O0-NEXT: v_sub_u32_e64 v12, s6, v4 +; GFX9-O0-NEXT: v_lshlrev_b64 v[23:24], v12, v[19:20] +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v24 +; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v12 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v23 +; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v7 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v7 +; GFX9-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v4, s6 +; GFX9-O0-NEXT: v_sub_u32_e64 v5, v4, s6 +; GFX9-O0-NEXT: v_lshrrev_b64 v[23:24], v5, v[19:20] +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v24 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v12, s[4:5] +; GFX9-O0-NEXT: s_mov_b32 s6, 0 +; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v4, s6 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v22 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v12, s[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v23 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v21 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[6:7] +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 +; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[19:20] +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v5 +; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 +; GFX9-O0-NEXT: s_mov_b32 s8, s7 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, s8 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v12, v12, v15, s[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4 +; GFX9-O0-NEXT: s_mov_b32 s8, s6 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[4:5] +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v14 +; GFX9-O0-NEXT: s_mov_b64 s[8:9], -1 +; GFX9-O0-NEXT: s_mov_b32 s5, s8 +; GFX9-O0-NEXT: s_mov_b32 s4, s9 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v18 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, s5 +; GFX9-O0-NEXT: v_add_co_u32_e32 v12, vcc, v12, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, s4 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v17, vcc, v15, v17, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v15, s5 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v14, vcc, v14, v15, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v15, s4 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v13, vcc, v13, v15, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v13 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v17 +; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v15, s9 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, s7 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, s6 +; GFX9-O0-NEXT: v_writelane_b32 v16, s4, 8 +; GFX9-O0-NEXT: v_writelane_b32 v16, s5, 9 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_branch .LBB1_6 +; GFX9-O0-NEXT: .LBB1_8: ; %udiv-bb1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 s[6:7], 1 +; GFX9-O0-NEXT: s_mov_b32 s5, s6 +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v1 +; GFX9-O0-NEXT: s_mov_b32 s4, s7 +; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 +; GFX9-O0-NEXT: s_mov_b32 s8, s6 +; GFX9-O0-NEXT: s_mov_b32 s9, s7 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s5 +; GFX9-O0-NEXT: v_add_co_u32_e32 v9, vcc, v4, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s4 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v5, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s9 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v5, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v2 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b32 s4, 0x7f +; GFX9-O0-NEXT: v_sub_u32_e64 v3, s4, v4 +; GFX9-O0-NEXT: v_lshlrev_b64 v[5:6], v3, v[11:12] +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v6 +; GFX9-O0-NEXT: s_mov_b32 s4, 64 +; GFX9-O0-NEXT: v_sub_u32_e64 v14, s4, v3 +; GFX9-O0-NEXT: v_lshrrev_b64 v[14:15], v14, v[7:8] +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v15 +; GFX9-O0-NEXT: v_or_b32_e64 v13, v13, v16 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v14 +; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v6 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v6 +; GFX9-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v3, s4 +; GFX9-O0-NEXT: s_mov_b32 s10, 63 +; GFX9-O0-NEXT: v_sub_u32_e64 v4, s10, v4 +; GFX9-O0-NEXT: v_lshlrev_b64 v[13:14], v4, v[7:8] +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v15, s[4:5] +; GFX9-O0-NEXT: s_mov_b32 s10, 0 +; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[10:11], v3, s10 +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v15, s[10:11] +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v13 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[10:11] +; GFX9-O0-NEXT: ; implicit-def: $sgpr10 +; GFX9-O0-NEXT: ; implicit-def: $sgpr10 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v4 +; GFX9-O0-NEXT: v_lshlrev_b64 v[7:8], v3, v[7:8] +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, s9 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[4:5] +; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr7_vgpr8 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, v4, v7, s[4:5] +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v3 +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v10 +; GFX9-O0-NEXT: v_or_b32_e64 v3, v3, v4 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v9 +; GFX9-O0-NEXT: v_or_b32_e64 v1, v1, v2 +; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[4:5], v[1:2], s[6:7] +; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v3, s8 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, s9 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, s6 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, s7 +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 s[6:7], exec +; GFX9-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5] +; GFX9-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7] +; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 6 +; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 7 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-O0-NEXT: s_cbranch_execz .LBB1_5 +; GFX9-O0-NEXT: s_branch .LBB1_7 +; GFX9-O0-NEXT: .LBB1_9: ; %udiv-end +; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b32 s4, 32 +; GFX9-O0-NEXT: s_waitcnt vmcnt(2) +; GFX9-O0-NEXT: v_lshrrev_b64 v[2:3], s4, v[7:8] +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v13 +; GFX9-O0-NEXT: v_mul_lo_u32 v5, v6, v2 +; GFX9-O0-NEXT: v_lshrrev_b64 v[13:14], s4, v[13:14] +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13 +; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr7_vgpr8 killed $exec +; GFX9-O0-NEXT: v_mul_lo_u32 v3, v7, v3 +; GFX9-O0-NEXT: v_mad_u64_u32 v[13:14], s[6:7], v7, v2, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v14 +; GFX9-O0-NEXT: v_add3_u32 v2, v2, v3, v5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-O0-NEXT: v_lshlrev_b64 v[17:18], s4, v[2:3] +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v18 +; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 killed $vgpr13_vgpr14 killed $exec +; GFX9-O0-NEXT: s_mov_b32 s5, 0 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v14 +; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13 +; GFX9-O0-NEXT: v_or_b32_e64 v13, v3, v5 +; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v2 +; GFX9-O0-NEXT: v_lshrrev_b64 v[2:3], s4, v[15:16] +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v11 +; GFX9-O0-NEXT: v_mul_lo_u32 v3, v2, v8 +; GFX9-O0-NEXT: v_lshrrev_b64 v[11:12], s4, v[11:12] +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 killed $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v15 +; GFX9-O0-NEXT: v_mul_lo_u32 v11, v11, v5 +; GFX9-O0-NEXT: v_mad_u64_u32 v[15:16], s[6:7], v2, v5, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v16 +; GFX9-O0-NEXT: v_add3_u32 v2, v2, v3, v11 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s6 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 +; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[2:3] +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v3 +; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 killed $vgpr15_vgpr16 killed $exec +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v16 +; GFX9-O0-NEXT: v_or_b32_e64 v11, v11, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v15 +; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v14 +; GFX9-O0-NEXT: v_add_co_u32_e64 v13, s[6:7], v11, v12 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v2, s[6:7], v2, v3, s[6:7] +; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v2 +; GFX9-O0-NEXT: v_mad_u64_u32 v[15:16], s[6:7], v8, v6, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v15 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v16 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, s6 +; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v12 +; GFX9-O0-NEXT: v_lshlrev_b64 v[15:16], s4, v[15:16] +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v16 +; GFX9-O0-NEXT: v_or_b32_e64 v11, v11, v12 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v15 +; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 +; GFX9-O0-NEXT: v_mad_u64_u32 v[15:16], s[6:7], v8, v7, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v15 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v16 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, s6 +; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v17 +; GFX9-O0-NEXT: v_lshlrev_b64 v[15:16], s4, v[15:16] +; GFX9-O0-NEXT: v_mov_b32_e32 v17, v16 +; GFX9-O0-NEXT: v_or_b32_e64 v8, v8, v17 +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 killed $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v15 +; GFX9-O0-NEXT: v_or_b32_e64 v19, v11, v12 +; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v8 +; GFX9-O0-NEXT: v_mad_u64_u32 v[11:12], s[6:7], v5, v7, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, v12 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v19 +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v20 +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v18 +; GFX9-O0-NEXT: v_add_co_u32_e64 v7, s[6:7], v7, v16 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v15, s[6:7], v8, v15, s[6:7] +; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v15 +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v8 +; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0xffffffff +; GFX9-O0-NEXT: s_mov_b32 s8, s7 +; GFX9-O0-NEXT: v_and_b32_e64 v15, v15, s8 +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v7 +; GFX9-O0-NEXT: ; kill: def $sgpr6 killed $sgpr6 killed $sgpr6_sgpr7 +; GFX9-O0-NEXT: v_and_b32_e64 v17, v16, s6 +; GFX9-O0-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v15 +; GFX9-O0-NEXT: v_mad_u64_u32 v[15:16], s[6:7], v5, v6, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v15 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v20 +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v16 +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: ; implicit-def: $sgpr7 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, s6 +; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v6 +; GFX9-O0-NEXT: v_lshlrev_b64 v[15:16], s4, v[15:16] +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v16 +; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v19 +; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 killed $vgpr15_vgpr16 killed $exec +; GFX9-O0-NEXT: v_or_b32_e64 v19, v6, v15 +; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v19 +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v20 +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v18 +; GFX9-O0-NEXT: v_add_co_u32_e64 v5, s[6:7], v5, v16 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v15, s[6:7], v6, v15, s[6:7] +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v15 +; GFX9-O0-NEXT: v_lshrrev_b64 v[17:18], s4, v[5:6] +; GFX9-O0-NEXT: v_lshrrev_b64 v[7:8], s4, v[7:8] +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v17 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v18 +; GFX9-O0-NEXT: v_add_co_u32_e64 v15, s[6:7], v15, v16 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v7, v8, s[6:7] +; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v15 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v16 +; GFX9-O0-NEXT: v_add_co_u32_e64 v15, s[6:7], v7, v8 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v2, s[6:7], v2, v3, s[6:7] +; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v15 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v16 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v14 +; GFX9-O0-NEXT: v_add_co_u32_e64 v2, s[6:7], v2, v8 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v3, v7, s[6:7] +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v7 +; GFX9-O0-NEXT: v_lshlrev_b64 v[6:7], s4, v[5:6] +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v7 +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 killed $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: ; implicit-def: $sgpr6 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s5 +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 +; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 +; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v7 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v10 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v7, vcc, v7, v8 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v6, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v5, vcc, v3, v5, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v2, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v1 +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr5 +; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5 +; GFX9-O0-NEXT: v_lshrrev_b64 v[7:8], s4, v[7:8] +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 +; GFX9-O0-NEXT: v_lshrrev_b64 v[5:6], s4, v[5:6] +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-O0-NEXT: ; kill: killed $vgpr4 +; GFX9-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_nop 0 +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: s_setpc_b64 s[30:31] + %div = urem i128 %lhs, %rhs + ret i128 %div +} + +define i128 @v_srem_i128_v_pow2k(i128 %lhs) { +; GFX9-LABEL: v_srem_i128_v_pow2k: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_ashrrev_i32_e32 v4, 31, v3 +; GFX9-NEXT: v_mov_b32_e32 v5, v4 +; GFX9-NEXT: v_lshrrev_b64 v[4:5], 31, v[4:5] +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v0, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v1, v5, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v2, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v3, vcc +; GFX9-NEXT: v_and_b32_e32 v4, -2, v4 +; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, 0, v0 +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v4, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v5, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v6, vcc +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-O0-LABEL: v_srem_i128_v_pow2k: +; GFX9-O0: ; %bb.0: +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v1 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v2 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 +; GFX9-O0-NEXT: s_mov_b32 s4, 63 +; GFX9-O0-NEXT: v_ashrrev_i64 v[6:7], s4, v[6:7] +; GFX9-O0-NEXT: s_mov_b32 s4, 31 +; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], s4, v[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v7 +; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 +; GFX9-O0-NEXT: s_mov_b32 s5, s6 +; GFX9-O0-NEXT: s_mov_b32 s4, s7 +; GFX9-O0-NEXT: v_add_co_u32_e32 v6, vcc, v5, v4 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v4, vcc, v0, v2, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v2, s5 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v8, vcc, v3, v2, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-O0-NEXT: v_addc_co_u32_e32 v2, vcc, v1, v2, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v4 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v7 +; GFX9-O0-NEXT: s_mov_b32 s6, -2 +; GFX9-O0-NEXT: s_mov_b32 s4, 0 +; GFX9-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5 +; GFX9-O0-NEXT: s_mov_b32 s5, s6 +; GFX9-O0-NEXT: s_mov_b32 s6, s5 +; GFX9-O0-NEXT: v_and_b32_e64 v4, v4, s6 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5 +; GFX9-O0-NEXT: v_and_b32_e64 v9, v6, s4 +; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v4 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v9 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v5, vcc, v5, v7 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v6, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v4, vcc +; GFX9-O0-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v2, vcc +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v1 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: s_mov_b32 s4, 32 +; GFX9-O0-NEXT: v_lshrrev_b64 v[5:6], s4, v[5:6] +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-O0-NEXT: v_lshrrev_b64 v[3:4], s4, v[3:4] +; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr3_vgpr4 killed $exec +; GFX9-O0-NEXT: s_setpc_b64 s[30:31] + %div = srem i128 %lhs, 8589934592 + ret i128 %div +} + +define i128 @v_urem_i128_v_pow2k(i128 %lhs) { +; GFX9-LABEL: v_urem_i128_v_pow2k: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_and_b32_e32 v1, 1, v1 +; GFX9-NEXT: v_mov_b32_e32 v2, 0 +; GFX9-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-O0-LABEL: v_urem_i128_v_pow2k: +; GFX9-O0: ; %bb.0: +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v1 +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec +; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr1 killed $exec +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4 +; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v2 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 +; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 +; GFX9-O0-NEXT: s_mov_b32 s6, 1 +; GFX9-O0-NEXT: s_mov_b32 s4, -1 +; GFX9-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5 +; GFX9-O0-NEXT: s_mov_b32 s5, s6 +; GFX9-O0-NEXT: s_mov_b32 s6, s5 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v1 +; GFX9-O0-NEXT: v_and_b32_e64 v3, v2, s6 +; GFX9-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5 +; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec +; GFX9-O0-NEXT: v_and_b32_e64 v1, v0, s4 +; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: s_mov_b32 s4, 32 +; GFX9-O0-NEXT: v_lshrrev_b64 v[1:2], s4, v[1:2] +; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: s_setpc_b64 s[30:31] + %div = urem i128 %lhs, 8589934592 + ret i128 %div +} + +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GFX9-SDAG: {{.*}} +; GFX9-SDAG-O0: {{.*}} diff --git a/llvm/test/CodeGen/ARM/cmse-vlldm-no-reorder.mir b/llvm/test/CodeGen/ARM/cmse-vlldm-no-reorder.mir index d006aa9..2bc4288 100644 --- a/llvm/test/CodeGen/ARM/cmse-vlldm-no-reorder.mir +++ b/llvm/test/CodeGen/ARM/cmse-vlldm-no-reorder.mir @@ -89,7 +89,7 @@ body: | # CHECK: $sp = t2STMDB_UPD $sp, 14 /* CC::al */, $noreg, $r4, $r5, $r6, undef $r7, $r8, $r9, $r10, $r11 # CHECK-NEXT: $r0 = t2BICri $r0, 1, 14 /* CC::al */, $noreg, $noreg # CHECK-NEXT: $sp = tSUBspi $sp, 34, 14 /* CC::al */, $noreg -# CHECK-NEXT: VLSTM $sp, 14 /* CC::al */, $noreg, 0, implicit-def $vpr, implicit-def $fpscr, implicit-def $fpscr_nzcv, implicit undef $vpr, implicit undef $fpscr, implicit undef $fpscr_nzcv, implicit undef $d0, implicit undef $d1, implicit undef $d2, implicit undef $d3, implicit undef $d4, implicit undef $d5, implicit undef $d6, implicit undef $d7, implicit $d8, implicit $d9, implicit $d10, implicit $d11, implicit $d12, implicit $d13, implicit $d14, implicit $d15 +# CHECK-NEXT: VLSTM $sp, 14 /* CC::al */, $noreg, implicit undef $vpr, implicit undef $fpscr, implicit undef $fpscr_nzcv, implicit undef $q0, implicit undef $q1, implicit undef $q2, implicit undef $q3, implicit undef $q4, implicit undef $q5, implicit undef $q6, implicit undef $q7 # CHECK-NEXT: $r1 = tMOVr $r0, 14 /* CC::al */, $noreg # CHECK-NEXT: $r2 = tMOVr $r0, 14 /* CC::al */, $noreg # CHECK-NEXT: $r3 = tMOVr $r0, 14 /* CC::al */, $noreg @@ -105,7 +105,7 @@ body: | # CHECK-NEXT: t2MSR_M 3072, $r0, 14 /* CC::al */, $noreg, implicit-def $cpsr # CHECK-NEXT: tBLXNSr 14 /* CC::al */, $noreg, killed $r0, csr_aapcs, implicit-def $lr, implicit $sp, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $s0 # CHECK-NEXT: $r12 = VMOVRS $s0, 14 /* CC::al */, $noreg -# CHECK-NEXT: VLLDM $sp, 14 /* CC::al */, $noreg, 0, implicit-def $vpr, implicit-def $fpscr, implicit-def $fpscr_nzcv, implicit-def $d0, implicit-def $d1, implicit-def $d2, implicit-def $d3, implicit-def $d4, implicit-def $d5, implicit-def $d6, implicit-def $d7, implicit-def $d8, implicit-def $d9, implicit-def $d10, implicit-def $d11, implicit-def $d12, implicit-def $d13, implicit-def $d14, implicit-def $d15 +# CHECK-NEXT: VLLDM $sp, 14 /* CC::al */, $noreg, implicit-def $q0, implicit-def $q1, implicit-def $q2, implicit-def $q3, implicit-def $q4, implicit-def $q5, implicit-def $q6, implicit-def $q7, implicit-def $vpr, implicit-def $fpscr, implicit-def $fpscr_nzcv # CHECK-NEXT: $s0 = VMOVSR $r12, 14 /* CC::al */, $noreg # CHECK-NEXT: $sp = tADDspi $sp, 34, 14 /* CC::al */, $noreg # CHECK-NEXT: $sp = t2LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r4, def $r5, def $r6, def $r7, def $r8, def $r9, def $r10, def $r11 diff --git a/llvm/test/CodeGen/ARM/vlldm-vlstm-uops.mir b/llvm/test/CodeGen/ARM/vlldm-vlstm-uops.mir index ad53add..8c49a53 100644 --- a/llvm/test/CodeGen/ARM/vlldm-vlstm-uops.mir +++ b/llvm/test/CodeGen/ARM/vlldm-vlstm-uops.mir @@ -2,7 +2,7 @@ --- | target triple = "thumbv8m.main-arm-none-eabi" - define hidden void @foo(void ()* nocapture %baz) local_unnamed_addr #0 { + define hidden void @foo(ptr nocapture %baz) local_unnamed_addr #0 { entry: %call = call i32 @bar() #0 %tobool = icmp eq i32 %call, 0 @@ -55,13 +55,14 @@ body: | tBL 14, $noreg, @bar, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $r0 bb.2.land.end: - liveins: $r4, $vpr, $fpscr, $fpscr_nzcv, $d0, $d1, $d2, $d3, $d4, $d5, $d6, $d7, $d8, $d9, $d10, $d11, $d12, $d13, $d14, $d15 + liveins: $r4 + $sp = t2STMDB_UPD $sp, 14, $noreg, $r4, killed $r5, killed $r6, killed $r7, killed $r8, killed $r9, killed $r10, killed $r11 $r4 = t2BICri $r4, 1, 14, $noreg, $noreg $sp = tSUBspi $sp, 34, 14, $noreg - VLSTM $sp, 14, $noreg, 0, implicit-def $vpr, implicit-def $fpscr, implicit-def $fpscr_nzcv, implicit $vpr, implicit $fpscr, implicit $fpscr_nzcv, implicit $d0, implicit $d1, implicit $d2, implicit $d3, implicit $d4, implicit $d5, implicit $d6, implicit $d7, implicit $d8, implicit $d9, implicit $d10, implicit $d11, implicit $d12, implicit $d13, implicit $d14, implicit $d15 + VLSTM $sp, 14, $noreg tBLXNSr 14, $noreg, killed $r4, csr_aapcs, implicit-def $lr, implicit $sp, implicit-def dead $lr, implicit $sp, implicit-def $sp - VLLDM $sp, 14, $noreg, 0, implicit-def $vpr, implicit-def $fpscr, implicit-def $fpscr_nzcv, implicit-def $d0, implicit-def $d1, implicit-def $d2, implicit-def $d3, implicit-def $d4, implicit-def $d5, implicit-def $d6, implicit-def $d7, implicit-def $d8, implicit-def $d9, implicit-def $d10, implicit-def $d11, implicit-def $d12, implicit-def $d13, implicit-def $d14, implicit-def $d15 + VLLDM $sp, 14, $noreg, implicit-def $q0, implicit-def $q1, implicit-def $q2, implicit-def $q3, implicit-def $q4, implicit-def $q5, implicit-def $q6, implicit-def $q7, implicit-def $vpr, implicit-def $fpscr, implicit-def $fpscr_nzcv $sp = tADDspi $sp, 34, 14, $noreg $sp = t2LDMIA_UPD $sp, 14, $noreg, def $r4, def $r5, def $r6, def $r7, def $r8, def $r9, def $r10, def $r11 $sp = t2LDMIA_RET $sp, 14, $noreg, def $r4, def $pc diff --git a/llvm/test/CodeGen/PowerPC/crsave.ll b/llvm/test/CodeGen/PowerPC/crsave.ll index 81e7a0a..bde49d0 100644 --- a/llvm/test/CodeGen/PowerPC/crsave.ll +++ b/llvm/test/CodeGen/PowerPC/crsave.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -O0 -frame-pointer=all -mtriple=powerpc-unknown-linux-gnu -mcpu=g5 < %s | FileCheck %s -check-prefix=PPC32 ; RUN: llc -O0 -mtriple=powerpc64-unknown-linux-gnu -mcpu=g5 < %s | FileCheck %s -check-prefix=PPC64 ; RUN: llc -O0 -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs < %s | FileCheck %s -check-prefix=PPC64-ELFv2 @@ -5,6 +6,101 @@ declare void @foo() define i32 @test_cr2() nounwind uwtable { +; PPC32-LABEL: test_cr2: +; PPC32: # %bb.0: # %entry +; PPC32-NEXT: mflr 0 +; PPC32-NEXT: stwu 1, -32(1) +; PPC32-NEXT: stw 31, 28(1) +; PPC32-NEXT: stw 0, 36(1) +; PPC32-NEXT: .cfi_def_cfa_offset 32 +; PPC32-NEXT: .cfi_offset r31, -4 +; PPC32-NEXT: .cfi_offset lr, 4 +; PPC32-NEXT: mr 31, 1 +; PPC32-NEXT: .cfi_def_cfa_register r31 +; PPC32-NEXT: mfcr 12 +; PPC32-NEXT: stw 12, 24(31) +; PPC32-NEXT: li 3, 1 +; PPC32-NEXT: li 4, 2 +; PPC32-NEXT: li 5, 3 +; PPC32-NEXT: li 6, 0 +; PPC32-NEXT: #APP +; PPC32-EMPTY: +; PPC32-NEXT: mtcr 6 +; PPC32-NEXT: cmpw 2, 4, 3 +; PPC32-NEXT: mfcr 3 +; PPC32-NEXT: #NO_APP +; PPC32-NEXT: stw 3, 20(31) +; PPC32-NEXT: bl foo +; PPC32-NEXT: lwz 3, 20(31) +; PPC32-NEXT: lwz 12, 24(31) +; PPC32-NEXT: mtocrf 32, 12 +; PPC32-NEXT: lwz 0, 36(1) +; PPC32-NEXT: lwz 31, 28(1) +; PPC32-NEXT: addi 1, 1, 32 +; PPC32-NEXT: mtlr 0 +; PPC32-NEXT: blr +; +; PPC64-LABEL: test_cr2: +; PPC64: # %bb.0: # %entry +; PPC64-NEXT: mflr 0 +; PPC64-NEXT: mfcr 12 +; PPC64-NEXT: stw 12, 8(1) +; PPC64-NEXT: stdu 1, -128(1) +; PPC64-NEXT: std 0, 144(1) +; PPC64-NEXT: .cfi_def_cfa_offset 128 +; PPC64-NEXT: .cfi_offset lr, 16 +; PPC64-NEXT: .cfi_offset cr2, 8 +; PPC64-NEXT: li 3, 1 +; PPC64-NEXT: li 4, 2 +; PPC64-NEXT: li 5, 3 +; PPC64-NEXT: li 6, 0 +; PPC64-NEXT: #APP +; PPC64-EMPTY: +; PPC64-NEXT: mtcr 6 +; PPC64-NEXT: cmpw 2, 4, 3 +; PPC64-NEXT: mfcr 3 +; PPC64-NEXT: #NO_APP +; PPC64-NEXT: stw 3, 124(1) +; PPC64-NEXT: bl foo +; PPC64-NEXT: nop +; PPC64-NEXT: lwz 3, 124(1) +; PPC64-NEXT: addi 1, 1, 128 +; PPC64-NEXT: ld 0, 16(1) +; PPC64-NEXT: lwz 12, 8(1) +; PPC64-NEXT: mtocrf 32, 12 +; PPC64-NEXT: mtlr 0 +; PPC64-NEXT: blr +; +; PPC64-ELFv2-LABEL: test_cr2: +; PPC64-ELFv2: # %bb.0: # %entry +; PPC64-ELFv2-NEXT: mflr 0 +; PPC64-ELFv2-NEXT: mfocrf 12, 32 +; PPC64-ELFv2-NEXT: stw 12, 8(1) +; PPC64-ELFv2-NEXT: stdu 1, -112(1) +; PPC64-ELFv2-NEXT: std 0, 128(1) +; PPC64-ELFv2-NEXT: .cfi_def_cfa_offset 112 +; PPC64-ELFv2-NEXT: .cfi_offset lr, 16 +; PPC64-ELFv2-NEXT: .cfi_offset cr2, 8 +; PPC64-ELFv2-NEXT: li 3, 1 +; PPC64-ELFv2-NEXT: li 4, 2 +; PPC64-ELFv2-NEXT: li 5, 3 +; PPC64-ELFv2-NEXT: li 6, 0 +; PPC64-ELFv2-NEXT: #APP +; PPC64-ELFv2-EMPTY: +; PPC64-ELFv2-NEXT: mtcr 6 +; PPC64-ELFv2-NEXT: cmpw 2, 4, 3 +; PPC64-ELFv2-NEXT: mfcr 3 +; PPC64-ELFv2-NEXT: #NO_APP +; PPC64-ELFv2-NEXT: stw 3, 108(1) +; PPC64-ELFv2-NEXT: bl foo +; PPC64-ELFv2-NEXT: nop +; PPC64-ELFv2-NEXT: lwz 3, 108(1) +; PPC64-ELFv2-NEXT: addi 1, 1, 112 +; PPC64-ELFv2-NEXT: ld 0, 16(1) +; PPC64-ELFv2-NEXT: lwz 12, 8(1) +; PPC64-ELFv2-NEXT: mtocrf 32, 12 +; PPC64-ELFv2-NEXT: mtlr 0 +; PPC64-ELFv2-NEXT: blr entry: %ret = alloca i32, align 4 %0 = call i32 asm sideeffect "\0A\09mtcr $4\0A\09cmpw 2,$2,$1\0A\09mfcr $0", "=r,r,r,r,r,~{cr2}"(i32 1, i32 2, i32 3, i32 0) nounwind @@ -14,27 +110,104 @@ entry: ret i32 %1 } -; PPC32-LABEL: test_cr2: -; PPC32: stwu 1, -32(1) -; PPC32: stw 31, 28(1) -; PPC32: mfcr 12 -; PPC32-NEXT: stw 12, 24(31) -; PPC32: lwz 12, 24(31) -; PPC32-NEXT: mtocrf 32, 12 - -; PPC64: .cfi_startproc -; PPC64: mfcr 12 -; PPC64: stw 12, 8(1) -; PPC64: stdu 1, -[[AMT:[0-9]+]](1) -; PPC64: .cfi_def_cfa_offset 128 -; PPC64: .cfi_offset lr, 16 -; PPC64: .cfi_offset cr2, 8 -; PPC64: addi 1, 1, [[AMT]] -; PPC64: lwz 12, 8(1) -; PPC64: mtocrf 32, 12 -; PPC64: .cfi_endproc - define i32 @test_cr234() nounwind { +; PPC32-LABEL: test_cr234: +; PPC32: # %bb.0: # %entry +; PPC32-NEXT: mflr 0 +; PPC32-NEXT: stwu 1, -32(1) +; PPC32-NEXT: stw 31, 28(1) +; PPC32-NEXT: stw 0, 36(1) +; PPC32-NEXT: mr 31, 1 +; PPC32-NEXT: mfcr 12 +; PPC32-NEXT: stw 12, 24(31) +; PPC32-NEXT: li 3, 1 +; PPC32-NEXT: li 4, 2 +; PPC32-NEXT: li 5, 3 +; PPC32-NEXT: li 6, 0 +; PPC32-NEXT: #APP +; PPC32-EMPTY: +; PPC32-NEXT: mtcr 6 +; PPC32-NEXT: cmpw 2, 4, 3 +; PPC32-NEXT: cmpw 3, 4, 4 +; PPC32-NEXT: cmpw 4, 4, 5 +; PPC32-NEXT: mfcr 3 +; PPC32-NEXT: #NO_APP +; PPC32-NEXT: stw 3, 20(31) +; PPC32-NEXT: bl foo +; PPC32-NEXT: lwz 3, 20(31) +; PPC32-NEXT: lwz 12, 24(31) +; PPC32-NEXT: mtocrf 32, 12 +; PPC32-NEXT: mtocrf 16, 12 +; PPC32-NEXT: mtocrf 8, 12 +; PPC32-NEXT: lwz 0, 36(1) +; PPC32-NEXT: lwz 31, 28(1) +; PPC32-NEXT: addi 1, 1, 32 +; PPC32-NEXT: mtlr 0 +; PPC32-NEXT: blr +; +; PPC64-LABEL: test_cr234: +; PPC64: # %bb.0: # %entry +; PPC64-NEXT: mflr 0 +; PPC64-NEXT: mfcr 12 +; PPC64-NEXT: stw 12, 8(1) +; PPC64-NEXT: stdu 1, -128(1) +; PPC64-NEXT: std 0, 144(1) +; PPC64-NEXT: li 3, 1 +; PPC64-NEXT: li 4, 2 +; PPC64-NEXT: li 5, 3 +; PPC64-NEXT: li 6, 0 +; PPC64-NEXT: #APP +; PPC64-EMPTY: +; PPC64-NEXT: mtcr 6 +; PPC64-NEXT: cmpw 2, 4, 3 +; PPC64-NEXT: cmpw 3, 4, 4 +; PPC64-NEXT: cmpw 4, 4, 5 +; PPC64-NEXT: mfcr 3 +; PPC64-NEXT: #NO_APP +; PPC64-NEXT: stw 3, 124(1) +; PPC64-NEXT: bl foo +; PPC64-NEXT: nop +; PPC64-NEXT: lwz 3, 124(1) +; PPC64-NEXT: addi 1, 1, 128 +; PPC64-NEXT: ld 0, 16(1) +; PPC64-NEXT: lwz 12, 8(1) +; PPC64-NEXT: mtocrf 32, 12 +; PPC64-NEXT: mtocrf 16, 12 +; PPC64-NEXT: mtocrf 8, 12 +; PPC64-NEXT: mtlr 0 +; PPC64-NEXT: blr +; +; PPC64-ELFv2-LABEL: test_cr234: +; PPC64-ELFv2: # %bb.0: # %entry +; PPC64-ELFv2-NEXT: mflr 0 +; PPC64-ELFv2-NEXT: mfcr 12 +; PPC64-ELFv2-NEXT: stw 12, 8(1) +; PPC64-ELFv2-NEXT: stdu 1, -112(1) +; PPC64-ELFv2-NEXT: std 0, 128(1) +; PPC64-ELFv2-NEXT: li 3, 1 +; PPC64-ELFv2-NEXT: li 4, 2 +; PPC64-ELFv2-NEXT: li 5, 3 +; PPC64-ELFv2-NEXT: li 6, 0 +; PPC64-ELFv2-NEXT: #APP +; PPC64-ELFv2-EMPTY: +; PPC64-ELFv2-NEXT: mtcr 6 +; PPC64-ELFv2-NEXT: cmpw 2, 4, 3 +; PPC64-ELFv2-NEXT: cmpw 3, 4, 4 +; PPC64-ELFv2-NEXT: cmpw 4, 4, 5 +; PPC64-ELFv2-NEXT: mfcr 3 +; PPC64-ELFv2-NEXT: #NO_APP +; PPC64-ELFv2-NEXT: stw 3, 108(1) +; PPC64-ELFv2-NEXT: bl foo +; PPC64-ELFv2-NEXT: nop +; PPC64-ELFv2-NEXT: lwz 3, 108(1) +; PPC64-ELFv2-NEXT: addi 1, 1, 112 +; PPC64-ELFv2-NEXT: ld 0, 16(1) +; PPC64-ELFv2-NEXT: lwz 12, 8(1) +; PPC64-ELFv2-NEXT: mtocrf 32, 12 +; PPC64-ELFv2-NEXT: mtocrf 16, 12 +; PPC64-ELFv2-NEXT: mtocrf 8, 12 +; PPC64-ELFv2-NEXT: mtlr 0 +; PPC64-ELFv2-NEXT: blr entry: %ret = alloca i32, align 4 %0 = call i32 asm sideeffect "\0A\09mtcr $4\0A\09cmpw 2,$2,$1\0A\09cmpw 3,$2,$2\0A\09cmpw 4,$2,$3\0A\09mfcr $0", "=r,r,r,r,r,~{cr2},~{cr3},~{cr4}"(i32 1, i32 2, i32 3, i32 0) nounwind @@ -44,41 +217,102 @@ entry: ret i32 %1 } -; PPC32-LABEL: test_cr234: -; PPC32: stwu 1, -32(1) -; PPC32: stw 31, 28(1) -; PPC32: mfcr 12 -; PPC32-NEXT: stw 12, 24(31) -; PPC32: lwz 12, 24(31) -; PPC32-NEXT: mtocrf 32, 12 -; PPC32-NEXT: mtocrf 16, 12 -; PPC32-NEXT: mtocrf 8, 12 - -; PPC64: mfcr 12 -; PPC64: stw 12, 8(1) -; PPC64: stdu 1, -[[AMT:[0-9]+]](1) -; PPC64: addi 1, 1, [[AMT]] -; PPC64: lwz 12, 8(1) -; PPC64: mtocrf 32, 12 -; PPC64: mtocrf 16, 12 -; PPC64: mtocrf 8, 12 - ; Generate mfocrf in prologue when we need to save 1 nonvolatile CR field define void @cloberOneNvCrField() { +; PPC32-LABEL: cloberOneNvCrField: +; PPC32: # %bb.0: # %entry +; PPC32-NEXT: stwu 1, -32(1) +; PPC32-NEXT: stw 31, 28(1) +; PPC32-NEXT: .cfi_def_cfa_offset 32 +; PPC32-NEXT: .cfi_offset r31, -4 +; PPC32-NEXT: mr 31, 1 +; PPC32-NEXT: .cfi_def_cfa_register r31 +; PPC32-NEXT: mfcr 12 +; PPC32-NEXT: stw 12, 24(31) +; PPC32-NEXT: #APP +; PPC32-NEXT: # clobbers +; PPC32-NEXT: #NO_APP +; PPC32-NEXT: lwz 12, 24(31) +; PPC32-NEXT: mtocrf 32, 12 +; PPC32-NEXT: lwz 31, 28(1) +; PPC32-NEXT: addi 1, 1, 32 +; PPC32-NEXT: blr +; +; PPC64-LABEL: cloberOneNvCrField: +; PPC64: # %bb.0: # %entry +; PPC64-NEXT: mfcr 12 +; PPC64-NEXT: stw 12, 8(1) +; PPC64-NEXT: #APP +; PPC64-NEXT: # clobbers +; PPC64-NEXT: #NO_APP +; PPC64-NEXT: lwz 12, 8(1) +; PPC64-NEXT: mtocrf 32, 12 +; PPC64-NEXT: blr +; +; PPC64-ELFv2-LABEL: cloberOneNvCrField: +; PPC64-ELFv2: # %bb.0: # %entry +; PPC64-ELFv2-NEXT: mfocrf 12, 32 +; PPC64-ELFv2-NEXT: stw 12, 8(1) +; PPC64-ELFv2-NEXT: #APP +; PPC64-ELFv2-NEXT: # clobbers +; PPC64-ELFv2-NEXT: #NO_APP +; PPC64-ELFv2-NEXT: lwz 12, 8(1) +; PPC64-ELFv2-NEXT: mtocrf 32, 12 +; PPC64-ELFv2-NEXT: blr entry: tail call void asm sideeffect "# clobbers", "~{cr2}"() ret void - -; PPC64-ELFv2-LABEL: @cloberOneNvCrField -; PPC64-ELFv2: mfocrf [[REG1:[0-9]+]], 32 } ; Generate mfcr in prologue when we need to save all nonvolatile CR field define void @cloberAllNvCrField() { +; PPC32-LABEL: cloberAllNvCrField: +; PPC32: # %bb.0: # %entry +; PPC32-NEXT: stwu 1, -32(1) +; PPC32-NEXT: stw 31, 28(1) +; PPC32-NEXT: .cfi_def_cfa_offset 32 +; PPC32-NEXT: .cfi_offset r31, -4 +; PPC32-NEXT: mr 31, 1 +; PPC32-NEXT: .cfi_def_cfa_register r31 +; PPC32-NEXT: mfcr 12 +; PPC32-NEXT: stw 12, 24(31) +; PPC32-NEXT: #APP +; PPC32-NEXT: # clobbers +; PPC32-NEXT: #NO_APP +; PPC32-NEXT: lwz 12, 24(31) +; PPC32-NEXT: mtocrf 32, 12 +; PPC32-NEXT: mtocrf 16, 12 +; PPC32-NEXT: mtocrf 8, 12 +; PPC32-NEXT: lwz 31, 28(1) +; PPC32-NEXT: addi 1, 1, 32 +; PPC32-NEXT: blr +; +; PPC64-LABEL: cloberAllNvCrField: +; PPC64: # %bb.0: # %entry +; PPC64-NEXT: mfcr 12 +; PPC64-NEXT: stw 12, 8(1) +; PPC64-NEXT: #APP +; PPC64-NEXT: # clobbers +; PPC64-NEXT: #NO_APP +; PPC64-NEXT: lwz 12, 8(1) +; PPC64-NEXT: mtocrf 32, 12 +; PPC64-NEXT: mtocrf 16, 12 +; PPC64-NEXT: mtocrf 8, 12 +; PPC64-NEXT: blr +; +; PPC64-ELFv2-LABEL: cloberAllNvCrField: +; PPC64-ELFv2: # %bb.0: # %entry +; PPC64-ELFv2-NEXT: mfcr 12 +; PPC64-ELFv2-NEXT: stw 12, 8(1) +; PPC64-ELFv2-NEXT: #APP +; PPC64-ELFv2-NEXT: # clobbers +; PPC64-ELFv2-NEXT: #NO_APP +; PPC64-ELFv2-NEXT: lwz 12, 8(1) +; PPC64-ELFv2-NEXT: mtocrf 32, 12 +; PPC64-ELFv2-NEXT: mtocrf 16, 12 +; PPC64-ELFv2-NEXT: mtocrf 8, 12 +; PPC64-ELFv2-NEXT: blr entry: tail call void asm sideeffect "# clobbers", "~{cr2},~{cr3},~{cr4}"() ret void - -; PPC64-ELFv2-LABEL: @cloberAllNvCrField -; PPC64-ELFv2: mfcr [[REG1:[0-9]+]] } diff --git a/llvm/test/CodeGen/RISCV/atomic-cmpxchg-branch-on-result.ll b/llvm/test/CodeGen/RISCV/atomic-cmpxchg-branch-on-result.ll index 90d7877..18b6649 100644 --- a/llvm/test/CodeGen/RISCV/atomic-cmpxchg-branch-on-result.ll +++ b/llvm/test/CodeGen/RISCV/atomic-cmpxchg-branch-on-result.ll @@ -1,13 +1,13 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=riscv32 -mattr=+a -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=NOZACAS,RV32IA %s -; RUN: llc -mtriple=riscv32 -mattr=+a,+experimental-zacas -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv32 -mattr=+a,+zacas -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=ZACAS,RV32IA-ZACAS %s ; RUN: llc -mtriple=riscv64 -mattr=+a -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=NOZACAS,RV64IA %s -; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-zacas -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv64 -mattr=+a,+zacas -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=ZACAS,RV64IA-ZACAS %s -; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-zacas,+experimental-zabha -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv64 -mattr=+a,+zacas,+experimental-zabha -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=ZACAS,RV64IA-ZABHA %s ; Test cmpxchg followed by a branch on the cmpxchg success value to see if the diff --git a/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll b/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll index 8df37bf..394dffa 100644 --- a/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll +++ b/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll @@ -3,25 +3,25 @@ ; RUN: | FileCheck -check-prefix=RV32I %s ; RUN: llc -mtriple=riscv32 -mattr=+a -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV32IA,RV32IA-WMO %s -; RUN: llc -mtriple=riscv32 -mattr=+a,+experimental-zacas -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv32 -mattr=+a,+zacas -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV32IA,RV32IA-ZACAS,RV32IA-WMO-ZACAS %s ; RUN: llc -mtriple=riscv32 -mattr=+a,+experimental-ztso -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV32IA,RV32IA-TSO %s -; RUN: llc -mtriple=riscv32 -mattr=+a,+experimental-ztso,+experimental-zacas -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv32 -mattr=+a,+experimental-ztso,+zacas -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV32IA,RV32IA-ZACAS,RV32IA-TSO-ZACAS %s ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefix=RV64I %s ; RUN: llc -mtriple=riscv64 -mattr=+a -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-WMO %s -; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-zacas -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv64 -mattr=+a,+zacas -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-ZACAS,RV64IA-WMO-ZACAS %s -; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-zacas,+experimental-zabha -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv64 -mattr=+a,+zacas,+experimental-zabha -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-ZABHA,RV64IA-WMO-ZABHA %s ; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-ztso -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-TSO %s -; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-ztso,+experimental-zacas -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-ztso,+zacas -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-ZACAS,RV64IA-TSO-ZACAS %s -; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-ztso,+experimental-zacas,+experimental-zabha -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-ztso,+zacas,+experimental-zabha -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-ZABHA,RV64IA-TSO-ZABHA %s define void @cmpxchg_i8_monotonic_monotonic(ptr %ptr, i8 %cmp, i8 %val) nounwind { diff --git a/llvm/test/CodeGen/RISCV/atomic-rmw.ll b/llvm/test/CodeGen/RISCV/atomic-rmw.ll index ee80250..fe53001 100644 --- a/llvm/test/CodeGen/RISCV/atomic-rmw.ll +++ b/llvm/test/CodeGen/RISCV/atomic-rmw.ll @@ -12,22 +12,22 @@ ; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-ztso -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-NOZACAS,RV64IA-TSO,RV64IA-TSO-NOZACAS %s -; RUN: llc -mtriple=riscv32 -mattr=+a,+experimental-zacas -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv32 -mattr=+a,+zacas -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV32IA,RV32IA-ZACAS,RV32IA-WMO,RV32IA-WMO-ZACAS %s -; RUN: llc -mtriple=riscv32 -mattr=+a,+experimental-ztso,+experimental-zacas -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv32 -mattr=+a,+experimental-ztso,+zacas -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV32IA,RV32IA-ZACAS,RV32IA-TSO,RV32IA-TSO-ZACAS %s -; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-zacas -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv64 -mattr=+a,+zacas -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-ZACAS,RV64IA-WMO,RV64IA-WMO-ZACAS %s -; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-ztso,+experimental-zacas -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-ztso,+zacas -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-ZACAS,RV64IA-TSO,RV64IA-TSO-ZACAS %s ; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-zabha -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-WMO,RV64IA-WMO-ZABHA,RV64IA-WMO-ZABHA-NOZACAS %s ; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-ztso,+experimental-zabha -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-TSO,RV64IA-TSO-ZABHA,RV64IA-TSO-ZABHA-NOZACAS %s -; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-zabha,+experimental-zacas -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-zabha,+zacas -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-WMO,RV64IA-WMO-ZABHA,RV64IA-WMO-ZABHA-ZACAS %s -; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-ztso,+experimental-zabha,+experimental-zacas -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-ztso,+experimental-zabha,+zacas -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-TSO,RV64IA-TSO-ZABHA,RV64IA-TSO-ZABHA-ZACAS %s define i8 @atomicrmw_xchg_i8_monotonic(ptr %a, i8 %b) nounwind { diff --git a/llvm/test/CodeGen/RISCV/atomic-signext.ll b/llvm/test/CodeGen/RISCV/atomic-signext.ll index 47807f7..bdf3b28 100644 --- a/llvm/test/CodeGen/RISCV/atomic-signext.ll +++ b/llvm/test/CodeGen/RISCV/atomic-signext.ll @@ -3,13 +3,13 @@ ; RUN: | FileCheck -check-prefix=RV32I %s ; RUN: llc -mtriple=riscv32 -mattr=+a -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV32IA,RV32IA-NOZACAS %s -; RUN: llc -mtriple=riscv32 -mattr=+a,+experimental-zacas -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv32 -mattr=+a,+zacas -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV32IA,RV32IA-ZACAS %s ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefix=RV64I %s ; RUN: llc -mtriple=riscv64 -mattr=+a -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-NOZACAS %s -; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-zacas -verify-machineinstrs < %s \ +; RUN: llc -mtriple=riscv64 -mattr=+a,+zacas -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefixes=RV64IA,RV64IA-ZACAS %s define signext i8 @atomic_load_i8_unordered(ptr %a) nounwind { diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll index 13635a9..561b0f2 100644 --- a/llvm/test/CodeGen/RISCV/attributes.ll +++ b/llvm/test/CodeGen/RISCV/attributes.ll @@ -111,7 +111,7 @@ ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zvfbfmin %s -o - | FileCheck --check-prefixes=CHECK,RV32ZVFBFMIN %s ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zvfbfwma %s -o - | FileCheck --check-prefixes=CHECK,RV32ZVFBFWMA %s ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zaamo %s -o - | FileCheck --check-prefix=RV32ZAAMO %s -; RUN: llc -mtriple=riscv32 -mattr=+experimental-zacas %s -o - | FileCheck --check-prefix=RV32ZACAS %s +; RUN: llc -mtriple=riscv32 -mattr=+zacas %s -o - | FileCheck --check-prefix=RV32ZACAS %s ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zalasr %s -o - | FileCheck --check-prefix=RV32ZALASR %s ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zalrsc %s -o - | FileCheck --check-prefix=RV32ZALRSC %s ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zicfilp %s -o - | FileCheck --check-prefix=RV32ZICFILP %s @@ -240,7 +240,7 @@ ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zvfbfmin %s -o - | FileCheck --check-prefixes=CHECK,RV64ZVFBFMIN %s ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zvfbfwma %s -o - | FileCheck --check-prefixes=CHECK,RV64ZVFBFWMA %s ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zaamo %s -o - | FileCheck --check-prefix=RV64ZAAMO %s -; RUN: llc -mtriple=riscv64 -mattr=+experimental-zacas %s -o - | FileCheck --check-prefix=RV64ZACAS %s +; RUN: llc -mtriple=riscv64 -mattr=+zacas %s -o - | FileCheck --check-prefix=RV64ZACAS %s ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zalasr %s -o - | FileCheck --check-prefix=RV64ZALASR %s ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zalrsc %s -o - | FileCheck --check-prefix=RV64ZALRSC %s ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zicfilp %s -o - | FileCheck --check-prefix=RV64ZICFILP %s diff --git a/llvm/test/CodeGen/RISCV/machine-combiner.ll b/llvm/test/CodeGen/RISCV/machine-combiner.ll index 7c1792e..cfdefec 100644 --- a/llvm/test/CodeGen/RISCV/machine-combiner.ll +++ b/llvm/test/CodeGen/RISCV/machine-combiner.ll @@ -1096,10 +1096,10 @@ declare double @llvm.maxnum.f64(double, double) define double @test_fmadd_strategy(double %a0, double %a1, double %a2, double %a3, i64 %flag) { ; CHECK_LOCAL-LABEL: test_fmadd_strategy: ; CHECK_LOCAL: # %bb.0: # %entry -; CHECK_LOCAL-NEXT: fmv.d fa5, fa0 ; CHECK_LOCAL-NEXT: fsub.d fa4, fa0, fa1 -; CHECK_LOCAL-NEXT: fmul.d fa0, fa4, fa2 ; CHECK_LOCAL-NEXT: andi a0, a0, 1 +; CHECK_LOCAL-NEXT: fmv.d fa5, fa0 +; CHECK_LOCAL-NEXT: fmul.d fa0, fa4, fa2 ; CHECK_LOCAL-NEXT: beqz a0, .LBB76_2 ; CHECK_LOCAL-NEXT: # %bb.1: # %entry ; CHECK_LOCAL-NEXT: fmul.d fa4, fa5, fa1 @@ -1110,10 +1110,10 @@ define double @test_fmadd_strategy(double %a0, double %a1, double %a2, double %a ; ; CHECK_GLOBAL-LABEL: test_fmadd_strategy: ; CHECK_GLOBAL: # %bb.0: # %entry -; CHECK_GLOBAL-NEXT: fmv.d fa5, fa0 ; CHECK_GLOBAL-NEXT: fsub.d fa4, fa0, fa1 -; CHECK_GLOBAL-NEXT: fmul.d fa0, fa4, fa2 ; CHECK_GLOBAL-NEXT: andi a0, a0, 1 +; CHECK_GLOBAL-NEXT: fmv.d fa5, fa0 +; CHECK_GLOBAL-NEXT: fmul.d fa0, fa4, fa2 ; CHECK_GLOBAL-NEXT: beqz a0, .LBB76_2 ; CHECK_GLOBAL-NEXT: # %bb.1: # %entry ; CHECK_GLOBAL-NEXT: fmul.d fa5, fa5, fa1 diff --git a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll index 191f047..5d09c39 100644 --- a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll +++ b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll @@ -2849,6 +2849,498 @@ for.body: ; preds = %for.body.preheader, br i1 %cmp.not, label %for.cond.cleanup, label %for.body } +declare <4 x i32> @llvm.smin.v4i32(<4 x i32>, <4 x i32>) + +define void @sink_splat_min(ptr nocapture %a, i32 signext %x) { +; CHECK-LABEL: sink_splat_min: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a2, 1024 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB46_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vmin.vx v8, v8, a1 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a2, a2, 4 +; CHECK-NEXT: addi a0, a0, -16 +; CHECK-NEXT: bnez a2, .LBB46_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat) + store <4 x i32> %1, ptr %0, align 4 + %index.next = sub nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +define void @sink_splat_min_commute(ptr nocapture %a, i32 signext %x) { +; CHECK-LABEL: sink_splat_min_commute: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a2, 1024 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB47_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vmin.vx v8, v8, a1 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a2, a2, 4 +; CHECK-NEXT: addi a0, a0, -16 +; CHECK-NEXT: bnez a2, .LBB47_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %broadcast.splat, <4 x i32> %wide.load) + store <4 x i32> %1, ptr %0, align 4 + %index.next = sub nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +declare <4 x i32> @llvm.smax.v4i32(<4 x i32>, <4 x i32>) + +define void @sink_splat_max(ptr nocapture %a, i32 signext %x) { +; CHECK-LABEL: sink_splat_max: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a2, 1024 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB48_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vmax.vx v8, v8, a1 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a2, a2, 4 +; CHECK-NEXT: addi a0, a0, -16 +; CHECK-NEXT: bnez a2, .LBB48_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat) + store <4 x i32> %1, ptr %0, align 4 + %index.next = sub nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +define void @sink_splat_max_commute(ptr nocapture %a, i32 signext %x) { +; CHECK-LABEL: sink_splat_max_commute: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a2, 1024 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB49_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vmax.vx v8, v8, a1 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a2, a2, 4 +; CHECK-NEXT: addi a0, a0, -16 +; CHECK-NEXT: bnez a2, .LBB49_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %broadcast.splat, <4 x i32> %wide.load) + store <4 x i32> %1, ptr %0, align 4 + %index.next = sub nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +declare <4 x i32> @llvm.umin.v4i32(<4 x i32>, <4 x i32>) + +define void @sink_splat_umin(ptr nocapture %a, i32 signext %x) { +; CHECK-LABEL: sink_splat_umin: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a2, 1024 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB50_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vminu.vx v8, v8, a1 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a2, a2, 4 +; CHECK-NEXT: addi a0, a0, -16 +; CHECK-NEXT: bnez a2, .LBB50_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat) + store <4 x i32> %1, ptr %0, align 4 + %index.next = sub nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +define void @sink_splat_umin_commute(ptr nocapture %a, i32 signext %x) { +; CHECK-LABEL: sink_splat_umin_commute: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a2, 1024 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB51_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vminu.vx v8, v8, a1 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a2, a2, 4 +; CHECK-NEXT: addi a0, a0, -16 +; CHECK-NEXT: bnez a2, .LBB51_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %broadcast.splat, <4 x i32> %wide.load) + store <4 x i32> %1, ptr %0, align 4 + %index.next = sub nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +declare <4 x i32> @llvm.umax.v4i32(<4 x i32>, <4 x i32>) + +define void @sink_splat_umax(ptr nocapture %a, i32 signext %x) { +; CHECK-LABEL: sink_splat_umax: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a2, 1024 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB52_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vmaxu.vx v8, v8, a1 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a2, a2, 4 +; CHECK-NEXT: addi a0, a0, -16 +; CHECK-NEXT: bnez a2, .LBB52_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat) + store <4 x i32> %1, ptr %0, align 4 + %index.next = sub nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +define void @sink_splat_umax_commute(ptr nocapture %a, i32 signext %x) { +; CHECK-LABEL: sink_splat_umax_commute: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a2, 1024 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB53_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vmaxu.vx v8, v8, a1 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a2, a2, 4 +; CHECK-NEXT: addi a0, a0, -16 +; CHECK-NEXT: bnez a2, .LBB53_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %broadcast.splat, <4 x i32> %wide.load) + store <4 x i32> %1, ptr %0, align 4 + %index.next = sub nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>) + +define void @sink_splat_sadd_sat(ptr nocapture %a, i32 signext %x) { +; CHECK-LABEL: sink_splat_sadd_sat: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a2, 1 +; CHECK-NEXT: add a2, a0, a2 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB54_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsadd.vx v8, v8, a1 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a2, .LBB54_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +define void @sink_splat_sadd_sat_commute(ptr nocapture %a, i32 signext %x) { +; CHECK-LABEL: sink_splat_sadd_sat_commute: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a2, 1 +; CHECK-NEXT: add a2, a0, a2 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB55_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsadd.vx v8, v8, a1 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a2, .LBB55_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %broadcast.splat, <4 x i32> %wide.load) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>) + +define void @sink_splat_ssub_sat(ptr nocapture %a, i32 signext %x) { +; CHECK-LABEL: sink_splat_ssub_sat: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a2, 1024 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB56_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vssub.vx v8, v8, a1 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a2, a2, 4 +; CHECK-NEXT: addi a0, a0, -16 +; CHECK-NEXT: bnez a2, .LBB56_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat) + store <4 x i32> %1, ptr %0, align 4 + %index.next = sub nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>) + +define void @sink_splat_uadd_sat(ptr nocapture %a, i32 signext %x) { +; CHECK-LABEL: sink_splat_uadd_sat: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a2, 1 +; CHECK-NEXT: add a2, a0, a2 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB57_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsaddu.vx v8, v8, a1 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a2, .LBB57_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +define void @sink_splat_uadd_sat_commute(ptr nocapture %a, i32 signext %x) { +; CHECK-LABEL: sink_splat_uadd_sat_commute: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a2, 1 +; CHECK-NEXT: add a2, a0, a2 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB58_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsaddu.vx v8, v8, a1 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a2, .LBB58_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %broadcast.splat, <4 x i32> %wide.load) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>) + +define void @sink_splat_usub_sat(ptr nocapture %a, i32 signext %x) { +; CHECK-LABEL: sink_splat_usub_sat: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a2, 1024 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB59_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vssubu.vx v8, v8, a1 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a2, a2, 4 +; CHECK-NEXT: addi a0, a0, -16 +; CHECK-NEXT: bnez a2, .LBB59_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat) + store <4 x i32> %1, ptr %0, align 4 + %index.next = sub nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + declare <4 x i32> @llvm.vp.mul.v4i32(<4 x i32>, <4 x i32>, <4 x i1>, i32) define void @sink_splat_vp_mul(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i32 zeroext %vl) { @@ -2857,7 +3349,7 @@ define void @sink_splat_vp_mul(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i3 ; CHECK-NEXT: lui a3, 1 ; CHECK-NEXT: add a3, a0, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB46_1: # %vector.body +; CHECK-NEXT: .LBB60_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma @@ -2865,7 +3357,7 @@ define void @sink_splat_vp_mul(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a3, .LBB46_1 +; CHECK-NEXT: bne a0, a3, .LBB60_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -2895,7 +3387,7 @@ define void @sink_splat_vp_add(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i3 ; CHECK-NEXT: lui a3, 1 ; CHECK-NEXT: add a3, a0, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB47_1: # %vector.body +; CHECK-NEXT: .LBB61_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma @@ -2903,7 +3395,7 @@ define void @sink_splat_vp_add(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a3, .LBB47_1 +; CHECK-NEXT: bne a0, a3, .LBB61_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -2931,7 +3423,7 @@ define void @sink_splat_vp_add_commute(ptr nocapture %a, i32 signext %x, <4 x i1 ; CHECK-NEXT: lui a3, 1 ; CHECK-NEXT: add a3, a0, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB48_1: # %vector.body +; CHECK-NEXT: .LBB62_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma @@ -2939,7 +3431,7 @@ define void @sink_splat_vp_add_commute(ptr nocapture %a, i32 signext %x, <4 x i1 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a3, .LBB48_1 +; CHECK-NEXT: bne a0, a3, .LBB62_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -2969,7 +3461,7 @@ define void @sink_splat_vp_sub(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i3 ; CHECK-NEXT: lui a3, 1 ; CHECK-NEXT: add a3, a0, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB49_1: # %vector.body +; CHECK-NEXT: .LBB63_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma @@ -2977,7 +3469,7 @@ define void @sink_splat_vp_sub(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a3, .LBB49_1 +; CHECK-NEXT: bne a0, a3, .LBB63_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3005,7 +3497,7 @@ define void @sink_splat_vp_rsub(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i ; CHECK-NEXT: lui a3, 1 ; CHECK-NEXT: add a3, a0, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB50_1: # %vector.body +; CHECK-NEXT: .LBB64_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma @@ -3013,7 +3505,7 @@ define void @sink_splat_vp_rsub(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a3, .LBB50_1 +; CHECK-NEXT: bne a0, a3, .LBB64_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3043,7 +3535,7 @@ define void @sink_splat_vp_shl(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i3 ; CHECK-NEXT: lui a3, 1 ; CHECK-NEXT: add a3, a0, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB51_1: # %vector.body +; CHECK-NEXT: .LBB65_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma @@ -3051,7 +3543,7 @@ define void @sink_splat_vp_shl(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a3, .LBB51_1 +; CHECK-NEXT: bne a0, a3, .LBB65_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3081,7 +3573,7 @@ define void @sink_splat_vp_lshr(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i ; CHECK-NEXT: lui a3, 1 ; CHECK-NEXT: add a3, a0, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB52_1: # %vector.body +; CHECK-NEXT: .LBB66_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma @@ -3089,7 +3581,7 @@ define void @sink_splat_vp_lshr(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a3, .LBB52_1 +; CHECK-NEXT: bne a0, a3, .LBB66_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3119,7 +3611,7 @@ define void @sink_splat_vp_ashr(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i ; CHECK-NEXT: lui a3, 1 ; CHECK-NEXT: add a3, a0, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB53_1: # %vector.body +; CHECK-NEXT: .LBB67_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma @@ -3127,7 +3619,7 @@ define void @sink_splat_vp_ashr(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a3, .LBB53_1 +; CHECK-NEXT: bne a0, a3, .LBB67_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3157,7 +3649,7 @@ define void @sink_splat_vp_fmul(ptr nocapture %a, float %x, <4 x i1> %m, i32 zer ; CHECK-NEXT: lui a2, 1 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB54_1: # %vector.body +; CHECK-NEXT: .LBB68_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma @@ -3165,7 +3657,7 @@ define void @sink_splat_vp_fmul(ptr nocapture %a, float %x, <4 x i1> %m, i32 zer ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a2, .LBB54_1 +; CHECK-NEXT: bne a0, a2, .LBB68_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3195,7 +3687,7 @@ define void @sink_splat_vp_fdiv(ptr nocapture %a, float %x, <4 x i1> %m, i32 zer ; CHECK-NEXT: lui a2, 1 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB55_1: # %vector.body +; CHECK-NEXT: .LBB69_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma @@ -3203,7 +3695,7 @@ define void @sink_splat_vp_fdiv(ptr nocapture %a, float %x, <4 x i1> %m, i32 zer ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a2, .LBB55_1 +; CHECK-NEXT: bne a0, a2, .LBB69_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3231,7 +3723,7 @@ define void @sink_splat_vp_frdiv(ptr nocapture %a, float %x, <4 x i1> %m, i32 ze ; CHECK-NEXT: lui a2, 1 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB56_1: # %vector.body +; CHECK-NEXT: .LBB70_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma @@ -3239,7 +3731,7 @@ define void @sink_splat_vp_frdiv(ptr nocapture %a, float %x, <4 x i1> %m, i32 ze ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a2, .LBB56_1 +; CHECK-NEXT: bne a0, a2, .LBB70_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3269,7 +3761,7 @@ define void @sink_splat_vp_fadd(ptr nocapture %a, float %x, <4 x i1> %m, i32 zer ; CHECK-NEXT: lui a2, 1 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB57_1: # %vector.body +; CHECK-NEXT: .LBB71_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma @@ -3277,7 +3769,7 @@ define void @sink_splat_vp_fadd(ptr nocapture %a, float %x, <4 x i1> %m, i32 zer ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a2, .LBB57_1 +; CHECK-NEXT: bne a0, a2, .LBB71_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3307,7 +3799,7 @@ define void @sink_splat_vp_fsub(ptr nocapture %a, float %x, <4 x i1> %m, i32 zer ; CHECK-NEXT: lui a2, 1 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB58_1: # %vector.body +; CHECK-NEXT: .LBB72_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma @@ -3315,7 +3807,7 @@ define void @sink_splat_vp_fsub(ptr nocapture %a, float %x, <4 x i1> %m, i32 zer ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a2, .LBB58_1 +; CHECK-NEXT: bne a0, a2, .LBB72_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3345,7 +3837,7 @@ define void @sink_splat_vp_frsub(ptr nocapture %a, float %x, <4 x i1> %m, i32 ze ; CHECK-NEXT: lui a2, 1 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB59_1: # %vector.body +; CHECK-NEXT: .LBB73_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma @@ -3353,7 +3845,7 @@ define void @sink_splat_vp_frsub(ptr nocapture %a, float %x, <4 x i1> %m, i32 ze ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a2, .LBB59_1 +; CHECK-NEXT: bne a0, a2, .LBB73_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3383,7 +3875,7 @@ define void @sink_splat_vp_udiv(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i ; CHECK-NEXT: lui a3, 1 ; CHECK-NEXT: add a3, a0, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB60_1: # %vector.body +; CHECK-NEXT: .LBB74_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma @@ -3391,7 +3883,7 @@ define void @sink_splat_vp_udiv(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a3, .LBB60_1 +; CHECK-NEXT: bne a0, a3, .LBB74_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3421,7 +3913,7 @@ define void @sink_splat_vp_sdiv(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i ; CHECK-NEXT: lui a3, 1 ; CHECK-NEXT: add a3, a0, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB61_1: # %vector.body +; CHECK-NEXT: .LBB75_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma @@ -3429,7 +3921,7 @@ define void @sink_splat_vp_sdiv(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a3, .LBB61_1 +; CHECK-NEXT: bne a0, a3, .LBB75_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3459,7 +3951,7 @@ define void @sink_splat_vp_urem(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i ; CHECK-NEXT: lui a3, 1 ; CHECK-NEXT: add a3, a0, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB62_1: # %vector.body +; CHECK-NEXT: .LBB76_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma @@ -3467,7 +3959,7 @@ define void @sink_splat_vp_urem(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a3, .LBB62_1 +; CHECK-NEXT: bne a0, a3, .LBB76_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3497,7 +3989,7 @@ define void @sink_splat_vp_srem(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i ; CHECK-NEXT: lui a3, 1 ; CHECK-NEXT: add a3, a0, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB63_1: # %vector.body +; CHECK-NEXT: .LBB77_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma @@ -3505,7 +3997,7 @@ define void @sink_splat_vp_srem(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a3, .LBB63_1 +; CHECK-NEXT: bne a0, a3, .LBB77_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3536,7 +4028,7 @@ define void @sink_splat_vp_srem_commute(ptr nocapture %a, i32 signext %x, <4 x i ; CHECK-NEXT: vmv.v.x v8, a1 ; CHECK-NEXT: lui a1, 1 ; CHECK-NEXT: add a1, a0, a1 -; CHECK-NEXT: .LBB64_1: # %vector.body +; CHECK-NEXT: .LBB78_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v9, (a0) ; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma @@ -3544,7 +4036,7 @@ define void @sink_splat_vp_srem_commute(ptr nocapture %a, i32 signext %x, <4 x i ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v9, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a1, .LBB64_1 +; CHECK-NEXT: bne a0, a1, .LBB78_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3574,7 +4066,7 @@ define void @sink_splat_vp_fma(ptr noalias nocapture %a, ptr nocapture readonly ; CHECK-NEXT: lui a3, 1 ; CHECK-NEXT: add a3, a1, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB65_1: # %vector.body +; CHECK-NEXT: .LBB79_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vle32.v v9, (a1) @@ -3584,7 +4076,7 @@ define void @sink_splat_vp_fma(ptr noalias nocapture %a, ptr nocapture readonly ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a1, a1, 16 ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a1, a3, .LBB65_1 +; CHECK-NEXT: bne a1, a3, .LBB79_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3614,7 +4106,7 @@ define void @sink_splat_vp_fma_commute(ptr noalias nocapture %a, ptr nocapture r ; CHECK-NEXT: lui a3, 1 ; CHECK-NEXT: add a3, a1, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: .LBB66_1: # %vector.body +; CHECK-NEXT: .LBB80_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vle32.v v9, (a1) @@ -3624,7 +4116,7 @@ define void @sink_splat_vp_fma_commute(ptr noalias nocapture %a, ptr nocapture r ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a1, a1, 16 ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a1, a3, .LBB66_1 +; CHECK-NEXT: bne a1, a3, .LBB80_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3655,13 +4147,13 @@ define void @sink_splat_mul_lmul2(ptr nocapture %a, i64 signext %x) { ; CHECK-NEXT: lui a2, 2 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-NEXT: .LBB67_1: # %vector.body +; CHECK-NEXT: .LBB81_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle64.v v8, (a0) ; CHECK-NEXT: vmul.vx v8, v8, a1 ; CHECK-NEXT: vse64.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 32 -; CHECK-NEXT: bne a0, a2, .LBB67_1 +; CHECK-NEXT: bne a0, a2, .LBB81_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3689,13 +4181,13 @@ define void @sink_splat_add_lmul2(ptr nocapture %a, i64 signext %x) { ; CHECK-NEXT: lui a2, 2 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-NEXT: .LBB68_1: # %vector.body +; CHECK-NEXT: .LBB82_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle64.v v8, (a0) ; CHECK-NEXT: vadd.vx v8, v8, a1 ; CHECK-NEXT: vse64.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 32 -; CHECK-NEXT: bne a0, a2, .LBB68_1 +; CHECK-NEXT: bne a0, a2, .LBB82_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3723,13 +4215,13 @@ define void @sink_splat_sub_lmul2(ptr nocapture %a, i64 signext %x) { ; CHECK-NEXT: lui a2, 2 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-NEXT: .LBB69_1: # %vector.body +; CHECK-NEXT: .LBB83_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle64.v v8, (a0) ; CHECK-NEXT: vsub.vx v8, v8, a1 ; CHECK-NEXT: vse64.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 32 -; CHECK-NEXT: bne a0, a2, .LBB69_1 +; CHECK-NEXT: bne a0, a2, .LBB83_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3757,13 +4249,13 @@ define void @sink_splat_rsub_lmul2(ptr nocapture %a, i64 signext %x) { ; CHECK-NEXT: lui a2, 2 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-NEXT: .LBB70_1: # %vector.body +; CHECK-NEXT: .LBB84_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle64.v v8, (a0) ; CHECK-NEXT: vrsub.vx v8, v8, a1 ; CHECK-NEXT: vse64.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 32 -; CHECK-NEXT: bne a0, a2, .LBB70_1 +; CHECK-NEXT: bne a0, a2, .LBB84_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3791,13 +4283,13 @@ define void @sink_splat_and_lmul2(ptr nocapture %a, i64 signext %x) { ; CHECK-NEXT: lui a2, 2 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-NEXT: .LBB71_1: # %vector.body +; CHECK-NEXT: .LBB85_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle64.v v8, (a0) ; CHECK-NEXT: vand.vx v8, v8, a1 ; CHECK-NEXT: vse64.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 32 -; CHECK-NEXT: bne a0, a2, .LBB71_1 +; CHECK-NEXT: bne a0, a2, .LBB85_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3825,13 +4317,13 @@ define void @sink_splat_or_lmul2(ptr nocapture %a, i64 signext %x) { ; CHECK-NEXT: lui a2, 2 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-NEXT: .LBB72_1: # %vector.body +; CHECK-NEXT: .LBB86_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle64.v v8, (a0) ; CHECK-NEXT: vor.vx v8, v8, a1 ; CHECK-NEXT: vse64.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 32 -; CHECK-NEXT: bne a0, a2, .LBB72_1 +; CHECK-NEXT: bne a0, a2, .LBB86_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3859,13 +4351,13 @@ define void @sink_splat_xor_lmul2(ptr nocapture %a, i64 signext %x) { ; CHECK-NEXT: lui a2, 2 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-NEXT: .LBB73_1: # %vector.body +; CHECK-NEXT: .LBB87_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle64.v v8, (a0) ; CHECK-NEXT: vxor.vx v8, v8, a1 ; CHECK-NEXT: vse64.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 32 -; CHECK-NEXT: bne a0, a2, .LBB73_1 +; CHECK-NEXT: bne a0, a2, .LBB87_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3894,13 +4386,13 @@ define void @sink_splat_mul_lmul8(ptr nocapture %a, i32 signext %x) { ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: li a3, 32 ; CHECK-NEXT: vsetvli zero, a3, e32, m8, ta, ma -; CHECK-NEXT: .LBB74_1: # %vector.body +; CHECK-NEXT: .LBB88_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vmul.vx v8, v8, a1 ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a2, .LBB74_1 +; CHECK-NEXT: bne a0, a2, .LBB88_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3929,13 +4421,13 @@ define void @sink_splat_add_lmul8(ptr nocapture %a, i32 signext %x) { ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: li a3, 32 ; CHECK-NEXT: vsetvli zero, a3, e32, m8, ta, ma -; CHECK-NEXT: .LBB75_1: # %vector.body +; CHECK-NEXT: .LBB89_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vadd.vx v8, v8, a1 ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a2, .LBB75_1 +; CHECK-NEXT: bne a0, a2, .LBB89_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3964,13 +4456,13 @@ define void @sink_splat_sub_lmul8(ptr nocapture %a, i32 signext %x) { ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: li a3, 32 ; CHECK-NEXT: vsetvli zero, a3, e32, m8, ta, ma -; CHECK-NEXT: .LBB76_1: # %vector.body +; CHECK-NEXT: .LBB90_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsub.vx v8, v8, a1 ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a2, .LBB76_1 +; CHECK-NEXT: bne a0, a2, .LBB90_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -3999,13 +4491,13 @@ define void @sink_splat_rsub_lmul8(ptr nocapture %a, i32 signext %x) { ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: li a3, 32 ; CHECK-NEXT: vsetvli zero, a3, e32, m8, ta, ma -; CHECK-NEXT: .LBB77_1: # %vector.body +; CHECK-NEXT: .LBB91_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vrsub.vx v8, v8, a1 ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a2, .LBB77_1 +; CHECK-NEXT: bne a0, a2, .LBB91_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -4034,13 +4526,13 @@ define void @sink_splat_and_lmul8(ptr nocapture %a, i32 signext %x) { ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: li a3, 32 ; CHECK-NEXT: vsetvli zero, a3, e32, m8, ta, ma -; CHECK-NEXT: .LBB78_1: # %vector.body +; CHECK-NEXT: .LBB92_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vand.vx v8, v8, a1 ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a2, .LBB78_1 +; CHECK-NEXT: bne a0, a2, .LBB92_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -4069,13 +4561,13 @@ define void @sink_splat_or_lmul8(ptr nocapture %a, i32 signext %x) { ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: li a3, 32 ; CHECK-NEXT: vsetvli zero, a3, e32, m8, ta, ma -; CHECK-NEXT: .LBB79_1: # %vector.body +; CHECK-NEXT: .LBB93_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vor.vx v8, v8, a1 ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a2, .LBB79_1 +; CHECK-NEXT: bne a0, a2, .LBB93_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -4104,13 +4596,13 @@ define void @sink_splat_xor_lmul8(ptr nocapture %a, i32 signext %x) { ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: li a3, 32 ; CHECK-NEXT: vsetvli zero, a3, e32, m8, ta, ma -; CHECK-NEXT: .LBB80_1: # %vector.body +; CHECK-NEXT: .LBB94_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vxor.vx v8, v8, a1 ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a2, .LBB80_1 +; CHECK-NEXT: bne a0, a2, .LBB94_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -4138,13 +4630,13 @@ define void @sink_splat_mul_lmulmf2(ptr nocapture %a, i32 signext %x) { ; CHECK-NEXT: lui a2, 2 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma -; CHECK-NEXT: .LBB81_1: # %vector.body +; CHECK-NEXT: .LBB95_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vmul.vx v8, v8, a1 ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 32 -; CHECK-NEXT: bne a0, a2, .LBB81_1 +; CHECK-NEXT: bne a0, a2, .LBB95_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -4172,13 +4664,13 @@ define void @sink_splat_add_lmulmf2(ptr nocapture %a, i32 signext %x) { ; CHECK-NEXT: lui a2, 2 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma -; CHECK-NEXT: .LBB82_1: # %vector.body +; CHECK-NEXT: .LBB96_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vadd.vx v8, v8, a1 ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 32 -; CHECK-NEXT: bne a0, a2, .LBB82_1 +; CHECK-NEXT: bne a0, a2, .LBB96_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -4206,13 +4698,13 @@ define void @sink_splat_sub_lmulmf2(ptr nocapture %a, i32 signext %x) { ; CHECK-NEXT: lui a2, 2 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma -; CHECK-NEXT: .LBB83_1: # %vector.body +; CHECK-NEXT: .LBB97_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vsub.vx v8, v8, a1 ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 32 -; CHECK-NEXT: bne a0, a2, .LBB83_1 +; CHECK-NEXT: bne a0, a2, .LBB97_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -4240,13 +4732,13 @@ define void @sink_splat_rsub_lmulmf2(ptr nocapture %a, i32 signext %x) { ; CHECK-NEXT: lui a2, 2 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma -; CHECK-NEXT: .LBB84_1: # %vector.body +; CHECK-NEXT: .LBB98_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vrsub.vx v8, v8, a1 ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 32 -; CHECK-NEXT: bne a0, a2, .LBB84_1 +; CHECK-NEXT: bne a0, a2, .LBB98_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -4274,13 +4766,13 @@ define void @sink_splat_and_lmulmf2(ptr nocapture %a, i32 signext %x) { ; CHECK-NEXT: lui a2, 2 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma -; CHECK-NEXT: .LBB85_1: # %vector.body +; CHECK-NEXT: .LBB99_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vand.vx v8, v8, a1 ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 32 -; CHECK-NEXT: bne a0, a2, .LBB85_1 +; CHECK-NEXT: bne a0, a2, .LBB99_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -4308,13 +4800,13 @@ define void @sink_splat_or_lmulmf2(ptr nocapture %a, i32 signext %x) { ; CHECK-NEXT: lui a2, 2 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma -; CHECK-NEXT: .LBB86_1: # %vector.body +; CHECK-NEXT: .LBB100_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vor.vx v8, v8, a1 ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 32 -; CHECK-NEXT: bne a0, a2, .LBB86_1 +; CHECK-NEXT: bne a0, a2, .LBB100_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -4342,13 +4834,13 @@ define void @sink_splat_xor_lmulmf2(ptr nocapture %a, i32 signext %x) { ; CHECK-NEXT: lui a2, 2 ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma -; CHECK-NEXT: .LBB87_1: # %vector.body +; CHECK-NEXT: .LBB101_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v8, (a0) ; CHECK-NEXT: vxor.vx v8, v8, a1 ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: addi a0, a0, 32 -; CHECK-NEXT: bne a0, a2, .LBB87_1 +; CHECK-NEXT: bne a0, a2, .LBB101_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -4380,7 +4872,7 @@ define void @sink_splat_vp_icmp(ptr nocapture %x, i32 signext %y, <4 x i1> %m, i ; CHECK-NEXT: add a3, a0, a3 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vmv.v.i v9, 0 -; CHECK-NEXT: .LBB88_1: # %vector.body +; CHECK-NEXT: .LBB102_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v10, (a0) ; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma @@ -4389,7 +4881,7 @@ define void @sink_splat_vp_icmp(ptr nocapture %x, i32 signext %y, <4 x i1> %m, i ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v9, (a0), v0.t ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a3, .LBB88_1 +; CHECK-NEXT: bne a0, a3, .LBB102_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -4421,7 +4913,7 @@ define void @sink_splat_vp_fcmp(ptr nocapture %x, float %y, <4 x i1> %m, i32 zer ; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vmv.v.i v9, 0 -; CHECK-NEXT: .LBB89_1: # %vector.body +; CHECK-NEXT: .LBB103_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vle32.v v10, (a0) ; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma @@ -4430,7 +4922,7 @@ define void @sink_splat_vp_fcmp(ptr nocapture %x, float %y, <4 x i1> %m, i32 zer ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v9, (a0), v0.t ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: bne a0, a2, .LBB89_1 +; CHECK-NEXT: bne a0, a2, .LBB103_1 ; CHECK-NEXT: # %bb.2: # %for.cond.cleanup ; CHECK-NEXT: ret entry: @@ -4451,3 +4943,485 @@ vector.body: ; preds = %vector.body, %entry for.cond.cleanup: ; preds = %vector.body ret void } + +declare <4 x i32> @llvm.vp.smin.v4i32(<4 x i32>, <4 x i32>, <4 x i1>, i32) + +define void @sink_splat_vp_min(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i32 zeroext %vl) { +; CHECK-LABEL: sink_splat_vp_min: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a3, 1 +; CHECK-NEXT: add a3, a0, a3 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB104_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma +; CHECK-NEXT: vmin.vx v8, v8, a1, v0.t +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a3, .LBB104_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.vp.smin.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat, <4 x i1> %m, i32 %vl) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +define void @sink_splat_vp_min_commute(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i32 zeroext %vl) { +; CHECK-LABEL: sink_splat_vp_min_commute: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a3, 1 +; CHECK-NEXT: add a3, a0, a3 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB105_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma +; CHECK-NEXT: vmin.vx v8, v8, a1, v0.t +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a3, .LBB105_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.vp.smin.v4i32(<4 x i32> %broadcast.splat, <4 x i32> %wide.load, <4 x i1> %m, i32 %vl) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +declare <4 x i32> @llvm.vp.smax.v4i32(<4 x i32>, <4 x i32>, <4 x i1>, i32) + +define void @sink_splat_vp_max(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i32 zeroext %vl) { +; CHECK-LABEL: sink_splat_vp_max: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a3, 1 +; CHECK-NEXT: add a3, a0, a3 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB106_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma +; CHECK-NEXT: vmax.vx v8, v8, a1, v0.t +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a3, .LBB106_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.vp.smax.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat, <4 x i1> %m, i32 %vl) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +define void @sink_splat_vp_max_commute(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i32 zeroext %vl) { +; CHECK-LABEL: sink_splat_vp_max_commute: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a3, 1 +; CHECK-NEXT: add a3, a0, a3 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB107_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma +; CHECK-NEXT: vmax.vx v8, v8, a1, v0.t +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a3, .LBB107_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.vp.smax.v4i32(<4 x i32> %broadcast.splat, <4 x i32> %wide.load, <4 x i1> %m, i32 %vl) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +define void @sink_splat_vp_umin_commute(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i32 zeroext %vl) { +; CHECK-LABEL: sink_splat_vp_umin_commute: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a3, 1 +; CHECK-NEXT: add a3, a0, a3 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB108_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma +; CHECK-NEXT: vminu.vx v8, v8, a1, v0.t +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a3, .LBB108_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.vp.umin.v4i32(<4 x i32> %broadcast.splat, <4 x i32> %wide.load, <4 x i1> %m, i32 %vl) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +declare <4 x i32> @llvm.vp.umax.v4i32(<4 x i32>, <4 x i32>, <4 x i1>, i32) + +define void @sink_splat_vp_umax(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i32 zeroext %vl) { +; CHECK-LABEL: sink_splat_vp_umax: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a3, 1 +; CHECK-NEXT: add a3, a0, a3 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB109_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma +; CHECK-NEXT: vmaxu.vx v8, v8, a1, v0.t +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a3, .LBB109_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.vp.umax.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat, <4 x i1> %m, i32 %vl) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +define void @sink_splat_vp_umax_commute(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i32 zeroext %vl) { +; CHECK-LABEL: sink_splat_vp_umax_commute: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a3, 1 +; CHECK-NEXT: add a3, a0, a3 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB110_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma +; CHECK-NEXT: vmaxu.vx v8, v8, a1, v0.t +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a3, .LBB110_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.vp.umax.v4i32(<4 x i32> %broadcast.splat, <4 x i32> %wide.load, <4 x i1> %m, i32 %vl) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +declare <4 x i32> @llvm.vp.sadd.sat.v4i32(<4 x i32>, <4 x i32>, <4 x i1>, i32) + +define void @sink_splat_vp_sadd_sat(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i32 zeroext %vl) { +; CHECK-LABEL: sink_splat_vp_sadd_sat: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a3, 1 +; CHECK-NEXT: add a3, a0, a3 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB111_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma +; CHECK-NEXT: vsadd.vx v8, v8, a1, v0.t +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a3, .LBB111_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.vp.sadd.sat.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat, <4 x i1> %m, i32 %vl) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +define void @sink_splat_vp_sadd_sat_commute(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i32 zeroext %vl) { +; CHECK-LABEL: sink_splat_vp_sadd_sat_commute: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a3, 1 +; CHECK-NEXT: add a3, a0, a3 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB112_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma +; CHECK-NEXT: vsadd.vx v8, v8, a1, v0.t +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a3, .LBB112_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.vp.sadd.sat.v4i32(<4 x i32> %broadcast.splat, <4 x i32> %wide.load, <4 x i1> %m, i32 %vl) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +declare <4 x i32> @llvm.vp.ssub.sat.v4i32(<4 x i32>, <4 x i32>, <4 x i1>, i32) + +define void @sink_splat_vp_ssub_sat(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i32 zeroext %vl) { +; CHECK-LABEL: sink_splat_vp_ssub_sat: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a3, 1024 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB113_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma +; CHECK-NEXT: vssub.vx v8, v8, a1, v0.t +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a3, a3, 4 +; CHECK-NEXT: addi a0, a0, -16 +; CHECK-NEXT: bnez a3, .LBB113_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.vp.ssub.sat.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat, <4 x i1> %m, i32 %vl) + store <4 x i32> %1, ptr %0, align 4 + %index.next = sub nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +declare <4 x i32> @llvm.vp.uadd.sat.v4i32(<4 x i32>, <4 x i32>, <4 x i1>, i32) + +define void @sink_splat_vp_uadd_sat(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i32 zeroext %vl) { +; CHECK-LABEL: sink_splat_vp_uadd_sat: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a3, 1 +; CHECK-NEXT: add a3, a0, a3 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB114_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma +; CHECK-NEXT: vsaddu.vx v8, v8, a1, v0.t +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a3, .LBB114_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.vp.uadd.sat.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat, <4 x i1> %m, i32 %vl) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +define void @sink_splat_vp_uadd_sat_commute(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i32 zeroext %vl) { +; CHECK-LABEL: sink_splat_vp_uadd_sat_commute: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a3, 1 +; CHECK-NEXT: add a3, a0, a3 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB115_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma +; CHECK-NEXT: vsaddu.vx v8, v8, a1, v0.t +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a0, a0, 16 +; CHECK-NEXT: bne a0, a3, .LBB115_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.vp.uadd.sat.v4i32(<4 x i32> %broadcast.splat, <4 x i32> %wide.load, <4 x i1> %m, i32 %vl) + store <4 x i32> %1, ptr %0, align 4 + %index.next = add nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} + +declare <4 x i32> @llvm.vp.usub.sat.v4i32(<4 x i32>, <4 x i32>, <4 x i1>, i32) + +define void @sink_splat_vp_usub_sat(ptr nocapture %a, i32 signext %x, <4 x i1> %m, i32 zeroext %vl) { +; CHECK-LABEL: sink_splat_vp_usub_sat: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a3, 1024 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: .LBB116_1: # %vector.body +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma +; CHECK-NEXT: vssubu.vx v8, v8, a1, v0.t +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a3, a3, 4 +; CHECK-NEXT: addi a0, a0, -16 +; CHECK-NEXT: bnez a3, .LBB116_1 +; CHECK-NEXT: # %bb.2: # %for.cond.cleanup +; CHECK-NEXT: ret +entry: + %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] + %0 = getelementptr inbounds i32, ptr %a, i64 %index + %wide.load = load <4 x i32>, ptr %0, align 4 + %1 = call <4 x i32> @llvm.vp.usub.sat.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat, <4 x i1> %m, i32 %vl) + store <4 x i32> %1, ptr %0, align 4 + %index.next = sub nuw i64 %index, 4 + %2 = icmp eq i64 %index.next, 1024 + br i1 %2, label %for.cond.cleanup, label %vector.body + +for.cond.cleanup: ; preds = %vector.body + ret void +} diff --git a/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll b/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll index 87406f2..c0c11fe 100644 --- a/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll +++ b/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll @@ -813,24 +813,24 @@ define i64 @select_sll(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) { ; RV32SFB-NEXT: not a7, a2 ; RV32SFB-NEXT: srli a0, a0, 1 ; RV32SFB-NEXT: sll t0, a1, a2 -; RV32SFB-NEXT: srl a0, a0, a7 ; RV32SFB-NEXT: addi a2, a2, -32 +; RV32SFB-NEXT: srl a0, a0, a7 ; RV32SFB-NEXT: mv a1, a3 -; RV32SFB-NEXT: bgez a2, .LBB20_2 +; RV32SFB-NEXT: bltz a2, .LBB20_2 ; RV32SFB-NEXT: # %bb.1: # %entry -; RV32SFB-NEXT: or a1, t0, a0 +; RV32SFB-NEXT: li a3, 0 ; RV32SFB-NEXT: .LBB20_2: # %entry -; RV32SFB-NEXT: bltz a2, .LBB20_4 +; RV32SFB-NEXT: bgez a2, .LBB20_4 ; RV32SFB-NEXT: # %bb.3: # %entry -; RV32SFB-NEXT: li a3, 0 +; RV32SFB-NEXT: or a1, t0, a0 ; RV32SFB-NEXT: .LBB20_4: # %entry ; RV32SFB-NEXT: beqz a6, .LBB20_6 ; RV32SFB-NEXT: # %bb.5: # %entry -; RV32SFB-NEXT: mv a1, a5 +; RV32SFB-NEXT: mv a3, a4 ; RV32SFB-NEXT: .LBB20_6: # %entry ; RV32SFB-NEXT: beqz a6, .LBB20_8 ; RV32SFB-NEXT: # %bb.7: # %entry -; RV32SFB-NEXT: mv a3, a4 +; RV32SFB-NEXT: mv a1, a5 ; RV32SFB-NEXT: .LBB20_8: # %entry ; RV32SFB-NEXT: mv a0, a3 ; RV32SFB-NEXT: ret @@ -874,24 +874,24 @@ define i64 @select_srl(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) { ; RV32SFB-NEXT: not a7, a2 ; RV32SFB-NEXT: slli a1, a1, 1 ; RV32SFB-NEXT: srl t0, a0, a2 -; RV32SFB-NEXT: sll a1, a1, a7 ; RV32SFB-NEXT: addi a2, a2, -32 +; RV32SFB-NEXT: sll a1, a1, a7 ; RV32SFB-NEXT: mv a0, a3 -; RV32SFB-NEXT: bgez a2, .LBB21_2 +; RV32SFB-NEXT: bltz a2, .LBB21_2 ; RV32SFB-NEXT: # %bb.1: # %entry -; RV32SFB-NEXT: or a0, t0, a1 +; RV32SFB-NEXT: li a3, 0 ; RV32SFB-NEXT: .LBB21_2: # %entry -; RV32SFB-NEXT: bltz a2, .LBB21_4 +; RV32SFB-NEXT: bgez a2, .LBB21_4 ; RV32SFB-NEXT: # %bb.3: # %entry -; RV32SFB-NEXT: li a3, 0 +; RV32SFB-NEXT: or a0, t0, a1 ; RV32SFB-NEXT: .LBB21_4: # %entry ; RV32SFB-NEXT: beqz a6, .LBB21_6 ; RV32SFB-NEXT: # %bb.5: # %entry -; RV32SFB-NEXT: mv a0, a4 +; RV32SFB-NEXT: mv a3, a5 ; RV32SFB-NEXT: .LBB21_6: # %entry ; RV32SFB-NEXT: beqz a6, .LBB21_8 ; RV32SFB-NEXT: # %bb.7: # %entry -; RV32SFB-NEXT: mv a3, a5 +; RV32SFB-NEXT: mv a0, a4 ; RV32SFB-NEXT: .LBB21_8: # %entry ; RV32SFB-NEXT: mv a1, a3 ; RV32SFB-NEXT: ret @@ -935,24 +935,24 @@ define i64 @select_sra(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) { ; RV32SFB-NEXT: not a7, a2 ; RV32SFB-NEXT: slli t0, a1, 1 ; RV32SFB-NEXT: srl t1, a0, a2 -; RV32SFB-NEXT: sll a7, t0, a7 ; RV32SFB-NEXT: addi a2, a2, -32 +; RV32SFB-NEXT: sll a7, t0, a7 ; RV32SFB-NEXT: mv a0, a3 -; RV32SFB-NEXT: bgez a2, .LBB22_2 +; RV32SFB-NEXT: bltz a2, .LBB22_2 ; RV32SFB-NEXT: # %bb.1: # %entry -; RV32SFB-NEXT: or a0, t1, a7 +; RV32SFB-NEXT: srai a3, a1, 31 ; RV32SFB-NEXT: .LBB22_2: # %entry -; RV32SFB-NEXT: bltz a2, .LBB22_4 +; RV32SFB-NEXT: bgez a2, .LBB22_4 ; RV32SFB-NEXT: # %bb.3: # %entry -; RV32SFB-NEXT: srai a3, a1, 31 +; RV32SFB-NEXT: or a0, t1, a7 ; RV32SFB-NEXT: .LBB22_4: # %entry ; RV32SFB-NEXT: beqz a6, .LBB22_6 ; RV32SFB-NEXT: # %bb.5: # %entry -; RV32SFB-NEXT: mv a0, a4 +; RV32SFB-NEXT: mv a3, a5 ; RV32SFB-NEXT: .LBB22_6: # %entry ; RV32SFB-NEXT: beqz a6, .LBB22_8 ; RV32SFB-NEXT: # %bb.7: # %entry -; RV32SFB-NEXT: mv a3, a5 +; RV32SFB-NEXT: mv a0, a4 ; RV32SFB-NEXT: .LBB22_8: # %entry ; RV32SFB-NEXT: mv a1, a3 ; RV32SFB-NEXT: ret @@ -1088,11 +1088,11 @@ define i64 @select_andi(i64 %A, i64 %C, i1 zeroext %cond) { ; RV32SFB-NEXT: # %bb.1: # %entry ; RV32SFB-NEXT: andi a2, a0, 567 ; RV32SFB-NEXT: .LBB25_2: # %entry +; RV32SFB-NEXT: mv a0, a2 ; RV32SFB-NEXT: bnez a4, .LBB25_4 ; RV32SFB-NEXT: # %bb.3: # %entry ; RV32SFB-NEXT: li a1, 0 ; RV32SFB-NEXT: .LBB25_4: # %entry -; RV32SFB-NEXT: mv a0, a2 ; RV32SFB-NEXT: ret entry: %0 = and i64 %A, 567 @@ -1130,13 +1130,13 @@ define i64 @select_ori(i64 %A, i64 %C, i1 zeroext %cond) { ; ; RV32SFB-LABEL: select_ori: ; RV32SFB: # %bb.0: # %entry -; RV32SFB-NEXT: beqz a4, .LBB26_2 +; RV32SFB-NEXT: bnez a4, .LBB26_2 ; RV32SFB-NEXT: # %bb.1: # %entry -; RV32SFB-NEXT: mv a1, a3 +; RV32SFB-NEXT: ori a2, a0, 890 ; RV32SFB-NEXT: .LBB26_2: # %entry -; RV32SFB-NEXT: bnez a4, .LBB26_4 +; RV32SFB-NEXT: beqz a4, .LBB26_4 ; RV32SFB-NEXT: # %bb.3: # %entry -; RV32SFB-NEXT: ori a2, a0, 890 +; RV32SFB-NEXT: mv a1, a3 ; RV32SFB-NEXT: .LBB26_4: # %entry ; RV32SFB-NEXT: mv a0, a2 ; RV32SFB-NEXT: ret @@ -1176,13 +1176,13 @@ define i64 @select_xori(i64 %A, i64 %C, i1 zeroext %cond) { ; ; RV32SFB-LABEL: select_xori: ; RV32SFB: # %bb.0: # %entry -; RV32SFB-NEXT: beqz a4, .LBB27_2 +; RV32SFB-NEXT: bnez a4, .LBB27_2 ; RV32SFB-NEXT: # %bb.1: # %entry -; RV32SFB-NEXT: mv a1, a3 +; RV32SFB-NEXT: xori a2, a0, 321 ; RV32SFB-NEXT: .LBB27_2: # %entry -; RV32SFB-NEXT: bnez a4, .LBB27_4 +; RV32SFB-NEXT: beqz a4, .LBB27_4 ; RV32SFB-NEXT: # %bb.3: # %entry -; RV32SFB-NEXT: xori a2, a0, 321 +; RV32SFB-NEXT: mv a1, a3 ; RV32SFB-NEXT: .LBB27_4: # %entry ; RV32SFB-NEXT: mv a0, a2 ; RV32SFB-NEXT: ret @@ -1272,11 +1272,11 @@ define i64 @select_srli(i64 %A, i64 %C, i1 zeroext %cond) { ; RV32SFB-NEXT: mv a0, a2 ; RV32SFB-NEXT: bnez a4, .LBB29_2 ; RV32SFB-NEXT: # %bb.1: # %entry -; RV32SFB-NEXT: srli a0, a1, 3 +; RV32SFB-NEXT: li a3, 0 ; RV32SFB-NEXT: .LBB29_2: # %entry ; RV32SFB-NEXT: bnez a4, .LBB29_4 ; RV32SFB-NEXT: # %bb.3: # %entry -; RV32SFB-NEXT: li a3, 0 +; RV32SFB-NEXT: srli a0, a1, 3 ; RV32SFB-NEXT: .LBB29_4: # %entry ; RV32SFB-NEXT: mv a1, a3 ; RV32SFB-NEXT: ret diff --git a/llvm/test/CodeGen/X86/avx512-insert-extract.ll b/llvm/test/CodeGen/X86/avx512-insert-extract.ll index abfe3e6..3e40bfa 100644 --- a/llvm/test/CodeGen/X86/avx512-insert-extract.ll +++ b/llvm/test/CodeGen/X86/avx512-insert-extract.ll @@ -2171,19 +2171,13 @@ define void @test_concat_v2i1(ptr %arg, ptr %arg1, ptr %arg2) nounwind { ; KNL-LABEL: test_concat_v2i1: ; KNL: ## %bb.0: ; KNL-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero -; KNL-NEXT: vpextrw $0, %xmm0, %eax -; KNL-NEXT: movzwl %ax, %eax -; KNL-NEXT: vmovd %eax, %xmm1 -; KNL-NEXT: vcvtph2ps %xmm1, %xmm1 +; KNL-NEXT: vcvtph2ps %xmm0, %xmm1 ; KNL-NEXT: vmovss {{.*#+}} xmm2 = [6.0E+0,0.0E+0,0.0E+0,0.0E+0] ; KNL-NEXT: vucomiss %xmm2, %xmm1 ; KNL-NEXT: setb %al ; KNL-NEXT: andl $1, %eax ; KNL-NEXT: kmovw %eax, %k0 -; KNL-NEXT: vpsrld $16, %xmm0, %xmm0 -; KNL-NEXT: vpextrw $0, %xmm0, %eax -; KNL-NEXT: movzwl %ax, %eax -; KNL-NEXT: vmovd %eax, %xmm0 +; KNL-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] ; KNL-NEXT: vcvtph2ps %xmm0, %xmm0 ; KNL-NEXT: vucomiss %xmm2, %xmm0 ; KNL-NEXT: setb %al @@ -2212,19 +2206,13 @@ define void @test_concat_v2i1(ptr %arg, ptr %arg1, ptr %arg2) nounwind { ; SKX-LABEL: test_concat_v2i1: ; SKX: ## %bb.0: ; SKX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero -; SKX-NEXT: vpsrld $16, %xmm0, %xmm1 -; SKX-NEXT: vpextrw $0, %xmm1, %eax -; SKX-NEXT: movzwl %ax, %eax -; SKX-NEXT: vmovd %eax, %xmm1 +; SKX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[1,1,1,1,4,5,6,7] ; SKX-NEXT: vcvtph2ps %xmm1, %xmm1 ; SKX-NEXT: vmovss {{.*#+}} xmm2 = [6.0E+0,0.0E+0,0.0E+0,0.0E+0] ; SKX-NEXT: vucomiss %xmm2, %xmm1 ; SKX-NEXT: setb %al ; SKX-NEXT: kmovd %eax, %k0 ; SKX-NEXT: kshiftlb $1, %k0, %k0 -; SKX-NEXT: vpextrw $0, %xmm0, %eax -; SKX-NEXT: movzwl %ax, %eax -; SKX-NEXT: vmovd %eax, %xmm0 ; SKX-NEXT: vcvtph2ps %xmm0, %xmm0 ; SKX-NEXT: vucomiss %xmm2, %xmm0 ; SKX-NEXT: setb %al diff --git a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll index f5cca78..86ebb1e 100644 --- a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll +++ b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll @@ -1436,10 +1436,8 @@ define void @half_vec_compare(ptr %x, ptr %y) { ; KNL: ## %bb.0: ## %entry ; KNL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero ; KNL-NEXT: ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0x07] -; KNL-NEXT: vpsrld $16, %xmm0, %xmm1 ## encoding: [0xc5,0xf1,0x72,0xd0,0x10] -; KNL-NEXT: vpextrw $0, %xmm1, %eax ## encoding: [0xc5,0xf9,0xc5,0xc1,0x00] -; KNL-NEXT: movzwl %ax, %eax ## encoding: [0x0f,0xb7,0xc0] -; KNL-NEXT: vmovd %eax, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc8] +; KNL-NEXT: vpshuflw $85, %xmm0, %xmm1 ## encoding: [0xc5,0xfb,0x70,0xc8,0x55] +; KNL-NEXT: ## xmm1 = xmm0[1,1,1,1,4,5,6,7] ; KNL-NEXT: vcvtph2ps %xmm1, %xmm1 ## encoding: [0xc4,0xe2,0x79,0x13,0xc9] ; KNL-NEXT: xorl %eax, %eax ## encoding: [0x31,0xc0] ; KNL-NEXT: vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2] @@ -1449,9 +1447,6 @@ define void @half_vec_compare(ptr %x, ptr %y) { ; KNL-NEXT: movl $0, %edx ## encoding: [0xba,0x00,0x00,0x00,0x00] ; KNL-NEXT: cmovnel %ecx, %edx ## encoding: [0x0f,0x45,0xd1] ; KNL-NEXT: cmovpl %ecx, %edx ## encoding: [0x0f,0x4a,0xd1] -; KNL-NEXT: vpextrw $0, %xmm0, %edi ## encoding: [0xc5,0xf9,0xc5,0xf8,0x00] -; KNL-NEXT: movzwl %di, %edi ## encoding: [0x0f,0xb7,0xff] -; KNL-NEXT: vmovd %edi, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc7] ; KNL-NEXT: vcvtph2ps %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x13,0xc0] ; KNL-NEXT: vucomiss %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xc2] ; KNL-NEXT: cmovnel %ecx, %eax ## encoding: [0x0f,0x45,0xc1] @@ -1468,10 +1463,8 @@ define void @half_vec_compare(ptr %x, ptr %y) { ; AVX512BW: ## %bb.0: ## %entry ; AVX512BW-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero ; AVX512BW-NEXT: ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0x07] -; AVX512BW-NEXT: vpsrld $16, %xmm0, %xmm1 ## encoding: [0xc5,0xf1,0x72,0xd0,0x10] -; AVX512BW-NEXT: vpextrw $0, %xmm1, %eax ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc5,0xc1,0x00] -; AVX512BW-NEXT: movzwl %ax, %eax ## encoding: [0x0f,0xb7,0xc0] -; AVX512BW-NEXT: vmovd %eax, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc8] +; AVX512BW-NEXT: vpshuflw $85, %xmm0, %xmm1 ## encoding: [0xc5,0xfb,0x70,0xc8,0x55] +; AVX512BW-NEXT: ## xmm1 = xmm0[1,1,1,1,4,5,6,7] ; AVX512BW-NEXT: vcvtph2ps %xmm1, %xmm1 ## encoding: [0xc4,0xe2,0x79,0x13,0xc9] ; AVX512BW-NEXT: xorl %eax, %eax ## encoding: [0x31,0xc0] ; AVX512BW-NEXT: vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2] @@ -1481,9 +1474,6 @@ define void @half_vec_compare(ptr %x, ptr %y) { ; AVX512BW-NEXT: movl $0, %edx ## encoding: [0xba,0x00,0x00,0x00,0x00] ; AVX512BW-NEXT: cmovnel %ecx, %edx ## encoding: [0x0f,0x45,0xd1] ; AVX512BW-NEXT: cmovpl %ecx, %edx ## encoding: [0x0f,0x4a,0xd1] -; AVX512BW-NEXT: vpextrw $0, %xmm0, %edi ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc5,0xf8,0x00] -; AVX512BW-NEXT: movzwl %di, %edi ## encoding: [0x0f,0xb7,0xff] -; AVX512BW-NEXT: vmovd %edi, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc7] ; AVX512BW-NEXT: vcvtph2ps %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x13,0xc0] ; AVX512BW-NEXT: vucomiss %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xc2] ; AVX512BW-NEXT: cmovnel %ecx, %eax ## encoding: [0x0f,0x45,0xc1] @@ -1500,10 +1490,8 @@ define void @half_vec_compare(ptr %x, ptr %y) { ; SKX: ## %bb.0: ## %entry ; SKX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero ; SKX-NEXT: ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0x07] -; SKX-NEXT: vpsrld $16, %xmm0, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0x72,0xd0,0x10] -; SKX-NEXT: vpextrw $0, %xmm1, %eax ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc5,0xc1,0x00] -; SKX-NEXT: movzwl %ax, %eax ## encoding: [0x0f,0xb7,0xc0] -; SKX-NEXT: vmovd %eax, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc8] +; SKX-NEXT: vpshuflw $85, %xmm0, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xfb,0x70,0xc8,0x55] +; SKX-NEXT: ## xmm1 = xmm0[1,1,1,1,4,5,6,7] ; SKX-NEXT: vcvtph2ps %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x13,0xc9] ; SKX-NEXT: vxorps %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe8,0x57,0xd2] ; SKX-NEXT: vucomiss %xmm2, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xca] @@ -1512,9 +1500,6 @@ define void @half_vec_compare(ptr %x, ptr %y) { ; SKX-NEXT: orb %al, %cl ## encoding: [0x08,0xc1] ; SKX-NEXT: testb %cl, %cl ## encoding: [0x84,0xc9] ; SKX-NEXT: setne %al ## encoding: [0x0f,0x95,0xc0] -; SKX-NEXT: vpextrw $0, %xmm0, %ecx ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc5,0xc8,0x00] -; SKX-NEXT: movzwl %cx, %ecx ## encoding: [0x0f,0xb7,0xc9] -; SKX-NEXT: vmovd %ecx, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc1] ; SKX-NEXT: vcvtph2ps %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x13,0xc0] ; SKX-NEXT: vucomiss %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xc2] ; SKX-NEXT: setp %cl ## encoding: [0x0f,0x9a,0xc1] diff --git a/llvm/test/CodeGen/X86/avx512fp16-fp-logic.ll b/llvm/test/CodeGen/X86/avx512fp16-fp-logic.ll index e2ea897..f6fb2fc 100644 --- a/llvm/test/CodeGen/X86/avx512fp16-fp-logic.ll +++ b/llvm/test/CodeGen/X86/avx512fp16-fp-logic.ll @@ -92,7 +92,7 @@ define half @f6(half %x, i16 %y) { define half @f7(half %x) { ; CHECK-LABEL: f7: ; CHECK: # %bb.0: -; CHECK-NEXT: vmovsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-NEXT: vmovsh {{.*#+}} xmm1 = [1.7881E-7,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0] ; CHECK-NEXT: vandps %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retq %bc1 = bitcast half %x to i16 @@ -106,7 +106,7 @@ define half @f7(half %x) { define half @f8(half %x) { ; CHECK-LABEL: f8: ; CHECK: # %bb.0: -; CHECK-NEXT: vmovsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-NEXT: vmovsh {{.*#+}} xmm1 = [2.3842E-7,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0] ; CHECK-NEXT: vandps %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retq %bc1 = bitcast half %x to i16 @@ -171,7 +171,7 @@ define half @xor(half %x, half %y) { define half @f7_or(half %x) { ; CHECK-LABEL: f7_or: ; CHECK: # %bb.0: -; CHECK-NEXT: vmovsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-NEXT: vmovsh {{.*#+}} xmm1 = [1.7881E-7,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0] ; CHECK-NEXT: vorps %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retq %bc1 = bitcast half %x to i16 @@ -183,7 +183,7 @@ define half @f7_or(half %x) { define half @f7_xor(half %x) { ; CHECK-LABEL: f7_xor: ; CHECK: # %bb.0: -; CHECK-NEXT: vmovsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-NEXT: vmovsh {{.*#+}} xmm1 = [1.7881E-7,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0] ; CHECK-NEXT: vxorps %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retq %bc1 = bitcast half %x to i16 @@ -199,7 +199,7 @@ define half @f7_xor(half %x) { define half @movmsk(half %x) { ; CHECK-LABEL: movmsk: ; CHECK: # %bb.0: -; CHECK-NEXT: vmovsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-NEXT: vmovsh {{.*#+}} xmm1 = [-0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0] ; CHECK-NEXT: vandps %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retq %bc1 = bitcast half %x to i16 @@ -271,7 +271,7 @@ define half @fadd_bitcast_fneg(half %x, half %y) { define half @fsub_bitcast_fneg(half %x, half %y) { ; CHECK-LABEL: fsub_bitcast_fneg: ; CHECK: # %bb.0: -; CHECK-NEXT: vmovsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 +; CHECK-NEXT: vmovsh {{.*#+}} xmm2 = [NaN,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0] ; CHECK-NEXT: vxorps %xmm2, %xmm1, %xmm1 ; CHECK-NEXT: vsubsh %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retq diff --git a/llvm/test/CodeGen/X86/cmov-fp.ll b/llvm/test/CodeGen/X86/cmov-fp.ll index 26e720f..77665d0 100644 --- a/llvm/test/CodeGen/X86/cmov-fp.ll +++ b/llvm/test/CodeGen/X86/cmov-fp.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=i686-- -mcpu pentium4 < %s | FileCheck %s -check-prefix=SSE -; RUN: llc -mtriple=i686-- -mcpu pentium3 < %s | FileCheck %s -check-prefix=NOSSE2 -; RUN: llc -mtriple=i686-- -mcpu pentium2 < %s | FileCheck %s -check-prefix=NOSSE1 +; RUN: llc -mtriple=i686-- -mcpu pentium3 < %s | FileCheck %s -check-prefixes=NOSSE,NOSSE2 +; RUN: llc -mtriple=i686-- -mcpu pentium2 < %s | FileCheck %s -check-prefixes=NOSSE,NOSSE1 ; RUN: llc -mtriple=i686-- -mcpu pentium < %s | FileCheck %s -check-prefix=NOCMOV ; PR14035 @@ -27,27 +27,16 @@ define double @test1(i32 %a, i32 %b, double %x) nounwind { ; SSE-NEXT: popl %ebp ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test1: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovnbe %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test1: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovnbe %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test1: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldl {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovnbe %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test1: ; NOCMOV: # %bb.0: @@ -90,27 +79,16 @@ define double @test2(i32 %a, i32 %b, double %x) nounwind { ; SSE-NEXT: popl %ebp ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test2: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovnb %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test2: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovnb %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test2: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldl {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovnb %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test2: ; NOCMOV: # %bb.0: @@ -153,27 +131,16 @@ define double @test3(i32 %a, i32 %b, double %x) nounwind { ; SSE-NEXT: popl %ebp ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test3: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovb %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test3: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovb %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test3: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldl {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovb %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test3: ; NOCMOV: # %bb.0: @@ -216,27 +183,16 @@ define double @test4(i32 %a, i32 %b, double %x) nounwind { ; SSE-NEXT: popl %ebp ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test4: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovbe %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test4: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovbe %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test4: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldl {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovbe %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test4: ; NOCMOV: # %bb.0: @@ -279,31 +235,18 @@ define double @test5(i32 %a, i32 %b, double %x) nounwind { ; SSE-NEXT: popl %ebp ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test5: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: setg %al -; NOSSE2-NEXT: testb %al, %al -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovne %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test5: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: setg %al -; NOSSE1-NEXT: testb %al, %al -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovne %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test5: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldl {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: setg %al +; NOSSE-NEXT: testb %al, %al +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovne %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test5: ; NOCMOV: # %bb.0: @@ -346,31 +289,18 @@ define double @test6(i32 %a, i32 %b, double %x) nounwind { ; SSE-NEXT: popl %ebp ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test6: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: setge %al -; NOSSE2-NEXT: testb %al, %al -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovne %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test6: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: setge %al -; NOSSE1-NEXT: testb %al, %al -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovne %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test6: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldl {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: setge %al +; NOSSE-NEXT: testb %al, %al +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovne %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test6: ; NOCMOV: # %bb.0: @@ -413,31 +343,18 @@ define double @test7(i32 %a, i32 %b, double %x) nounwind { ; SSE-NEXT: popl %ebp ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test7: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: setl %al -; NOSSE2-NEXT: testb %al, %al -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovne %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test7: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: setl %al -; NOSSE1-NEXT: testb %al, %al -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovne %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test7: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldl {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: setl %al +; NOSSE-NEXT: testb %al, %al +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovne %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test7: ; NOCMOV: # %bb.0: @@ -480,31 +397,18 @@ define double @test8(i32 %a, i32 %b, double %x) nounwind { ; SSE-NEXT: popl %ebp ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test8: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: setle %al -; NOSSE2-NEXT: testb %al, %al -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovne %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test8: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldl {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: setle %al -; NOSSE1-NEXT: testb %al, %al -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovne %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test8: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldl {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: setle %al +; NOSSE-NEXT: testb %al, %al +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovne %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test8: ; NOCMOV: # %bb.0: @@ -1065,27 +969,16 @@ define x86_fp80 @test17(i32 %a, i32 %b, x86_fp80 %x) nounwind { ; SSE-NEXT: fstp %st(1) ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test17: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovnbe %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test17: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovnbe %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test17: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldt {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovnbe %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test17: ; NOCMOV: # %bb.0: @@ -1118,27 +1011,16 @@ define x86_fp80 @test18(i32 %a, i32 %b, x86_fp80 %x) nounwind { ; SSE-NEXT: fstp %st(1) ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test18: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovnb %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test18: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovnb %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test18: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldt {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovnb %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test18: ; NOCMOV: # %bb.0: @@ -1171,27 +1053,16 @@ define x86_fp80 @test19(i32 %a, i32 %b, x86_fp80 %x) nounwind { ; SSE-NEXT: fstp %st(1) ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test19: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovb %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test19: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovb %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test19: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldt {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovb %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test19: ; NOCMOV: # %bb.0: @@ -1224,27 +1095,16 @@ define x86_fp80 @test20(i32 %a, i32 %b, x86_fp80 %x) nounwind { ; SSE-NEXT: fstp %st(1) ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test20: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovbe %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test20: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovbe %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test20: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldt {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovbe %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test20: ; NOCMOV: # %bb.0: @@ -1279,31 +1139,18 @@ define x86_fp80 @test21(i32 %a, i32 %b, x86_fp80 %x) nounwind { ; SSE-NEXT: fstp %st(1) ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test21: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: setg %al -; NOSSE2-NEXT: testb %al, %al -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovne %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test21: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: setg %al -; NOSSE1-NEXT: testb %al, %al -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovne %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test21: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldt {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: setg %al +; NOSSE-NEXT: testb %al, %al +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovne %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test21: ; NOCMOV: # %bb.0: @@ -1339,31 +1186,18 @@ define x86_fp80 @test22(i32 %a, i32 %b, x86_fp80 %x) nounwind { ; SSE-NEXT: fstp %st(1) ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test22: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: setge %al -; NOSSE2-NEXT: testb %al, %al -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovne %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test22: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: setge %al -; NOSSE1-NEXT: testb %al, %al -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovne %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test22: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldt {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: setge %al +; NOSSE-NEXT: testb %al, %al +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovne %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test22: ; NOCMOV: # %bb.0: @@ -1398,31 +1232,18 @@ define x86_fp80 @test23(i32 %a, i32 %b, x86_fp80 %x) nounwind { ; SSE-NEXT: fstp %st(1) ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test23: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: setl %al -; NOSSE2-NEXT: testb %al, %al -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovne %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test23: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: setl %al -; NOSSE1-NEXT: testb %al, %al -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovne %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test23: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldt {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: setl %al +; NOSSE-NEXT: testb %al, %al +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovne %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test23: ; NOCMOV: # %bb.0: @@ -1457,31 +1278,18 @@ define x86_fp80 @test24(i32 %a, i32 %b, x86_fp80 %x) nounwind { ; SSE-NEXT: fstp %st(1) ; SSE-NEXT: retl ; -; NOSSE2-LABEL: test24: -; NOSSE2: # %bb.0: -; NOSSE2-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE2-NEXT: setle %al -; NOSSE2-NEXT: testb %al, %al -; NOSSE2-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE2-NEXT: fxch %st(1) -; NOSSE2-NEXT: fcmovne %st(1), %st -; NOSSE2-NEXT: fstp %st(1) -; NOSSE2-NEXT: retl -; -; NOSSE1-LABEL: test24: -; NOSSE1: # %bb.0: -; NOSSE1-NEXT: fldt {{[0-9]+}}(%esp) -; NOSSE1-NEXT: movl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: cmpl {{[0-9]+}}(%esp), %eax -; NOSSE1-NEXT: setle %al -; NOSSE1-NEXT: testb %al, %al -; NOSSE1-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; NOSSE1-NEXT: fxch %st(1) -; NOSSE1-NEXT: fcmovne %st(1), %st -; NOSSE1-NEXT: fstp %st(1) -; NOSSE1-NEXT: retl +; NOSSE-LABEL: test24: +; NOSSE: # %bb.0: +; NOSSE-NEXT: fldt {{[0-9]+}}(%esp) +; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; NOSSE-NEXT: setle %al +; NOSSE-NEXT: testb %al, %al +; NOSSE-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; NOSSE-NEXT: fxch %st(1) +; NOSSE-NEXT: fcmovne %st(1), %st +; NOSSE-NEXT: fstp %st(1) +; NOSSE-NEXT: retl ; ; NOCMOV-LABEL: test24: ; NOCMOV: # %bb.0: diff --git a/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll b/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll index 7039e33..cbb5bd09 100644 --- a/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll +++ b/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll @@ -160,6 +160,53 @@ define <16 x i8> @demandedelts_pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> ret <16 x i8> %5 } +define <4 x float> @demandedbits_sitofp_blendvps(<4 x float> %a0, <4 x float> %a1, <4 x i32> %a2) { +; SSE-LABEL: demandedbits_sitofp_blendvps: +; SSE: # %bb.0: +; SSE-NEXT: movaps %xmm0, %xmm3 +; SSE-NEXT: cvtdq2ps %xmm2, %xmm0 +; SSE-NEXT: blendvps %xmm0, %xmm1, %xmm3 +; SSE-NEXT: movaps %xmm3, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: demandedbits_sitofp_blendvps: +; AVX: # %bb.0: +; AVX-NEXT: vcvtdq2ps %xmm2, %xmm2 +; AVX-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %cvt = sitofp <4 x i32> %a2 to <4 x float> + %sel = tail call noundef <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %cvt) + ret <4 x float> %sel +} + +define <4 x float> @demandedbits_uitofp_blendvps(<4 x float> %a0, <4 x float> %a1, <4 x i32> %a2) { +; SSE-LABEL: demandedbits_uitofp_blendvps: +; SSE: # %bb.0: +; SSE-NEXT: movaps %xmm0, %xmm3 +; SSE-NEXT: movdqa {{.*#+}} xmm0 = [1258291200,1258291200,1258291200,1258291200] +; SSE-NEXT: pblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7] +; SSE-NEXT: psrld $16, %xmm2 +; SSE-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0],mem[1],xmm2[2],mem[3],xmm2[4],mem[5],xmm2[6],mem[7] +; SSE-NEXT: subps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 +; SSE-NEXT: addps %xmm2, %xmm0 +; SSE-NEXT: blendvps %xmm0, %xmm1, %xmm3 +; SSE-NEXT: movaps %xmm3, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: demandedbits_uitofp_blendvps: +; AVX: # %bb.0: +; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],mem[1],xmm2[2],mem[3],xmm2[4],mem[5],xmm2[6],mem[7] +; AVX-NEXT: vpsrld $16, %xmm2, %xmm2 +; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],mem[1],xmm2[2],mem[3],xmm2[4],mem[5],xmm2[6],mem[7] +; AVX-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 +; AVX-NEXT: vaddps %xmm2, %xmm3, %xmm2 +; AVX-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %cvt = uitofp <4 x i32> %a2 to <4 x float> + %sel = tail call noundef <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %cvt) + ret <4 x float> %sel +} + define <2 x i64> @demandedbits_blendvpd(i64 %a0, i64 %a2, <2 x double> %a3) { ; SSE-LABEL: demandedbits_blendvpd: ; SSE: # %bb.0: diff --git a/llvm/test/CodeGen/X86/cvt16.ll b/llvm/test/CodeGen/X86/cvt16.ll index 59097f8..c7ef353 100644 --- a/llvm/test/CodeGen/X86/cvt16.ll +++ b/llvm/test/CodeGen/X86/cvt16.ll @@ -89,7 +89,6 @@ define float @test3(float %src) nounwind uwtable readnone { ; F16C-LABEL: test3: ; F16C: # %bb.0: ; F16C-NEXT: vcvtps2ph $4, %xmm0, %xmm0 -; F16C-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero ; F16C-NEXT: vcvtph2ps %xmm0, %xmm0 ; F16C-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/f16c-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/f16c-intrinsics-fast-isel.ll index e114c20..1886e29 100644 --- a/llvm/test/CodeGen/X86/f16c-intrinsics-fast-isel.ll +++ b/llvm/test/CodeGen/X86/f16c-intrinsics-fast-isel.ll @@ -18,8 +18,7 @@ define float @test_cvtsh_ss(i16 %a0) nounwind { ; ; X64-LABEL: test_cvtsh_ss: ; X64: # %bb.0: -; X64-NEXT: movzwl %di, %eax -; X64-NEXT: vmovd %eax, %xmm0 +; X64-NEXT: vmovd %edi, %xmm0 ; X64-NEXT: vcvtph2ps %xmm0, %xmm0 ; X64-NEXT: retq %ins0 = insertelement <8 x i16> undef, i16 %a0, i32 0 @@ -41,8 +40,6 @@ define i16 @test_cvtss_sh(float %a0) nounwind { ; X86-LABEL: test_cvtss_sh: ; X86: # %bb.0: ; X86-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vxorps %xmm1, %xmm1, %xmm1 -; X86-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3] ; X86-NEXT: vcvtps2ph $0, %xmm0, %xmm0 ; X86-NEXT: vmovd %xmm0, %eax ; X86-NEXT: # kill: def $ax killed $ax killed $eax @@ -50,8 +47,6 @@ define i16 @test_cvtss_sh(float %a0) nounwind { ; ; X64-LABEL: test_cvtss_sh: ; X64: # %bb.0: -; X64-NEXT: vxorps %xmm1, %xmm1, %xmm1 -; X64-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3] ; X64-NEXT: vcvtps2ph $0, %xmm0, %xmm0 ; X64-NEXT: vmovd %xmm0, %eax ; X64-NEXT: # kill: def $ax killed $ax killed $eax diff --git a/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll b/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll index 5f326b6..8f875c7 100644 --- a/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll +++ b/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll @@ -1432,7 +1432,6 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bounds(i32 %cnt) nounwind { ; CHECK-NO-FASTFMA-NEXT: shll %cl, %eax ; CHECK-NO-FASTFMA-NEXT: vcvtusi2ss %eax, %xmm0, %xmm0 ; CHECK-NO-FASTFMA-NEXT: vcvtps2ph $4, %xmm0, %xmm0 -; CHECK-NO-FASTFMA-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero ; CHECK-NO-FASTFMA-NEXT: vcvtph2ps %xmm0, %xmm0 ; CHECK-NO-FASTFMA-NEXT: vmovss {{.*#+}} xmm1 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0] ; CHECK-NO-FASTFMA-NEXT: vdivss %xmm0, %xmm1, %xmm0 @@ -1447,7 +1446,6 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bounds(i32 %cnt) nounwind { ; CHECK-FMA-NEXT: shlxl %edi, %eax, %eax ; CHECK-FMA-NEXT: vcvtusi2ss %eax, %xmm0, %xmm0 ; CHECK-FMA-NEXT: vcvtps2ph $4, %xmm0, %xmm0 -; CHECK-FMA-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero ; CHECK-FMA-NEXT: vcvtph2ps %xmm0, %xmm0 ; CHECK-FMA-NEXT: vmovss {{.*#+}} xmm1 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0] ; CHECK-FMA-NEXT: vdivss %xmm0, %xmm1, %xmm0 @@ -1550,7 +1548,6 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bound2(i16 %cnt) nounwind { ; CHECK-NO-FASTFMA-NEXT: movzwl %ax, %eax ; CHECK-NO-FASTFMA-NEXT: vcvtsi2ss %eax, %xmm0, %xmm0 ; CHECK-NO-FASTFMA-NEXT: vcvtps2ph $4, %xmm0, %xmm0 -; CHECK-NO-FASTFMA-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero ; CHECK-NO-FASTFMA-NEXT: vcvtph2ps %xmm0, %xmm0 ; CHECK-NO-FASTFMA-NEXT: vmovss {{.*#+}} xmm1 = [2.0E+0,0.0E+0,0.0E+0,0.0E+0] ; CHECK-NO-FASTFMA-NEXT: vdivss %xmm0, %xmm1, %xmm0 @@ -1566,7 +1563,6 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bound2(i16 %cnt) nounwind { ; CHECK-FMA-NEXT: movzwl %ax, %eax ; CHECK-FMA-NEXT: vcvtsi2ss %eax, %xmm0, %xmm0 ; CHECK-FMA-NEXT: vcvtps2ph $4, %xmm0, %xmm0 -; CHECK-FMA-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero ; CHECK-FMA-NEXT: vcvtph2ps %xmm0, %xmm0 ; CHECK-FMA-NEXT: vmovss {{.*#+}} xmm1 = [2.0E+0,0.0E+0,0.0E+0,0.0E+0] ; CHECK-FMA-NEXT: vdivss %xmm0, %xmm1, %xmm0 diff --git a/llvm/test/CodeGen/X86/fp-roundeven.ll b/llvm/test/CodeGen/X86/fp-roundeven.ll index fed2060..8037c78 100644 --- a/llvm/test/CodeGen/X86/fp-roundeven.ll +++ b/llvm/test/CodeGen/X86/fp-roundeven.ll @@ -51,7 +51,6 @@ define half @roundeven_f16(half %h) { ; AVX512F-LABEL: roundeven_f16: ; AVX512F: ## %bb.0: ## %entry ; AVX512F-NEXT: vpextrw $0, %xmm0, %eax -; AVX512F-NEXT: movzwl %ax, %eax ; AVX512F-NEXT: vmovd %eax, %xmm0 ; AVX512F-NEXT: vcvtph2ps %xmm0, %xmm0 ; AVX512F-NEXT: vroundss $8, %xmm0, %xmm0, %xmm0 diff --git a/llvm/test/CodeGen/X86/fpclamptosat_vec.ll b/llvm/test/CodeGen/X86/fpclamptosat_vec.ll index c8708ea..6aad4c2 100644 --- a/llvm/test/CodeGen/X86/fpclamptosat_vec.ll +++ b/llvm/test/CodeGen/X86/fpclamptosat_vec.ll @@ -698,30 +698,18 @@ define <4 x i32> @stest_f16i32(<4 x half> %x) nounwind { ; ; AVX2-LABEL: stest_f16i32: ; AVX2: # %bb.0: # %entry -; AVX2-NEXT: vpsrlq $48, %xmm0, %xmm1 -; AVX2-NEXT: vpextrw $0, %xmm1, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm1 +; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[3,3,3,3,4,5,6,7] ; AVX2-NEXT: vcvtph2ps %xmm1, %xmm1 -; AVX2-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3] -; AVX2-NEXT: vpextrw $0, %xmm2, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm2 ; AVX2-NEXT: vcvttss2si %xmm1, %rax -; AVX2-NEXT: vcvtph2ps %xmm2, %xmm1 +; AVX2-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] +; AVX2-NEXT: vcvtph2ps %xmm1, %xmm1 +; AVX2-NEXT: vcvttss2si %xmm1, %rcx +; AVX2-NEXT: vcvtph2ps %xmm0, %xmm1 ; AVX2-NEXT: vmovq %rax, %xmm2 ; AVX2-NEXT: vcvttss2si %xmm1, %rax -; AVX2-NEXT: vmovq %rax, %xmm1 +; AVX2-NEXT: vmovq %rcx, %xmm1 ; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; AVX2-NEXT: vpextrw $0, %xmm0, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm2 -; AVX2-NEXT: vcvtph2ps %xmm2, %xmm2 -; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0 -; AVX2-NEXT: vpextrw $0, %xmm0, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm0 -; AVX2-NEXT: vcvttss2si %xmm2, %rax +; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] ; AVX2-NEXT: vcvtph2ps %xmm0, %xmm0 ; AVX2-NEXT: vmovq %rax, %xmm2 ; AVX2-NEXT: vcvttss2si %xmm0, %rax @@ -848,10 +836,7 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) nounwind { ; ; AVX2-LABEL: utesth_f16i32: ; AVX2: # %bb.0: # %entry -; AVX2-NEXT: vpsrlq $48, %xmm0, %xmm1 -; AVX2-NEXT: vpextrw $0, %xmm1, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm1 +; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[3,3,3,3,4,5,6,7] ; AVX2-NEXT: vcvtph2ps %xmm1, %xmm2 ; AVX2-NEXT: vmovss {{.*#+}} xmm1 = [9.22337203E+18,0.0E+0,0.0E+0,0.0E+0] ; AVX2-NEXT: vsubss %xmm1, %xmm2, %xmm3 @@ -862,20 +847,14 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) nounwind { ; AVX2-NEXT: andq %rax, %rdx ; AVX2-NEXT: orq %rcx, %rdx ; AVX2-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3] -; AVX2-NEXT: vpextrw $0, %xmm2, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm2 ; AVX2-NEXT: vcvtph2ps %xmm2, %xmm2 ; AVX2-NEXT: vsubss %xmm1, %xmm2, %xmm3 ; AVX2-NEXT: vcvttss2si %xmm3, %rax ; AVX2-NEXT: vcvttss2si %xmm2, %rcx ; AVX2-NEXT: vmovq %rdx, %xmm2 -; AVX2-NEXT: vpextrw $0, %xmm0, %edx -; AVX2-NEXT: movzwl %dx, %edx -; AVX2-NEXT: vmovd %edx, %xmm3 ; AVX2-NEXT: movq %rcx, %rdx ; AVX2-NEXT: sarq $63, %rdx -; AVX2-NEXT: vcvtph2ps %xmm3, %xmm3 +; AVX2-NEXT: vcvtph2ps %xmm0, %xmm3 ; AVX2-NEXT: andq %rax, %rdx ; AVX2-NEXT: vsubss %xmm1, %xmm3, %xmm4 ; AVX2-NEXT: vcvttss2si %xmm4, %rax @@ -887,10 +866,7 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) nounwind { ; AVX2-NEXT: sarq $63, %rdx ; AVX2-NEXT: andq %rax, %rdx ; AVX2-NEXT: orq %rcx, %rdx -; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0 -; AVX2-NEXT: vpextrw $0, %xmm0, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm0 +; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] ; AVX2-NEXT: vcvtph2ps %xmm0, %xmm0 ; AVX2-NEXT: vsubss %xmm1, %xmm0, %xmm1 ; AVX2-NEXT: vcvttss2si %xmm1, %rax @@ -1023,31 +999,19 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) nounwind { ; ; AVX2-LABEL: ustest_f16i32: ; AVX2: # %bb.0: # %entry -; AVX2-NEXT: vpsrlq $48, %xmm0, %xmm1 -; AVX2-NEXT: vpextrw $0, %xmm1, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm1 +; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[3,3,3,3,4,5,6,7] ; AVX2-NEXT: vcvtph2ps %xmm1, %xmm1 ; AVX2-NEXT: vcvttss2si %xmm1, %rax ; AVX2-NEXT: vmovq %rax, %xmm1 ; AVX2-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3] -; AVX2-NEXT: vpextrw $0, %xmm2, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm2 ; AVX2-NEXT: vcvtph2ps %xmm2, %xmm2 ; AVX2-NEXT: vcvttss2si %xmm2, %rax ; AVX2-NEXT: vmovq %rax, %xmm2 ; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX2-NEXT: vpextrw $0, %xmm0, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm2 -; AVX2-NEXT: vcvtph2ps %xmm2, %xmm2 +; AVX2-NEXT: vcvtph2ps %xmm0, %xmm2 ; AVX2-NEXT: vcvttss2si %xmm2, %rax ; AVX2-NEXT: vmovq %rax, %xmm2 -; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0 -; AVX2-NEXT: vpextrw $0, %xmm0, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm0 +; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] ; AVX2-NEXT: vcvtph2ps %xmm0, %xmm0 ; AVX2-NEXT: vcvttss2si %xmm0, %rax ; AVX2-NEXT: vmovq %rax, %xmm0 @@ -3346,30 +3310,18 @@ define <4 x i32> @stest_f16i32_mm(<4 x half> %x) nounwind { ; ; AVX2-LABEL: stest_f16i32_mm: ; AVX2: # %bb.0: # %entry -; AVX2-NEXT: vpsrlq $48, %xmm0, %xmm1 -; AVX2-NEXT: vpextrw $0, %xmm1, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm1 +; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[3,3,3,3,4,5,6,7] ; AVX2-NEXT: vcvtph2ps %xmm1, %xmm1 -; AVX2-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3] -; AVX2-NEXT: vpextrw $0, %xmm2, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm2 ; AVX2-NEXT: vcvttss2si %xmm1, %rax -; AVX2-NEXT: vcvtph2ps %xmm2, %xmm1 +; AVX2-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] +; AVX2-NEXT: vcvtph2ps %xmm1, %xmm1 +; AVX2-NEXT: vcvttss2si %xmm1, %rcx +; AVX2-NEXT: vcvtph2ps %xmm0, %xmm1 ; AVX2-NEXT: vmovq %rax, %xmm2 ; AVX2-NEXT: vcvttss2si %xmm1, %rax -; AVX2-NEXT: vmovq %rax, %xmm1 +; AVX2-NEXT: vmovq %rcx, %xmm1 ; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; AVX2-NEXT: vpextrw $0, %xmm0, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm2 -; AVX2-NEXT: vcvtph2ps %xmm2, %xmm2 -; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0 -; AVX2-NEXT: vpextrw $0, %xmm0, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm0 -; AVX2-NEXT: vcvttss2si %xmm2, %rax +; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] ; AVX2-NEXT: vcvtph2ps %xmm0, %xmm0 ; AVX2-NEXT: vmovq %rax, %xmm2 ; AVX2-NEXT: vcvttss2si %xmm0, %rax @@ -3494,10 +3446,7 @@ define <4 x i32> @utesth_f16i32_mm(<4 x half> %x) nounwind { ; ; AVX2-LABEL: utesth_f16i32_mm: ; AVX2: # %bb.0: # %entry -; AVX2-NEXT: vpsrlq $48, %xmm0, %xmm1 -; AVX2-NEXT: vpextrw $0, %xmm1, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm1 +; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[3,3,3,3,4,5,6,7] ; AVX2-NEXT: vcvtph2ps %xmm1, %xmm2 ; AVX2-NEXT: vmovss {{.*#+}} xmm1 = [9.22337203E+18,0.0E+0,0.0E+0,0.0E+0] ; AVX2-NEXT: vsubss %xmm1, %xmm2, %xmm3 @@ -3508,20 +3457,14 @@ define <4 x i32> @utesth_f16i32_mm(<4 x half> %x) nounwind { ; AVX2-NEXT: andq %rax, %rdx ; AVX2-NEXT: orq %rcx, %rdx ; AVX2-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3] -; AVX2-NEXT: vpextrw $0, %xmm2, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm2 ; AVX2-NEXT: vcvtph2ps %xmm2, %xmm2 ; AVX2-NEXT: vsubss %xmm1, %xmm2, %xmm3 ; AVX2-NEXT: vcvttss2si %xmm3, %rax ; AVX2-NEXT: vcvttss2si %xmm2, %rcx ; AVX2-NEXT: vmovq %rdx, %xmm2 -; AVX2-NEXT: vpextrw $0, %xmm0, %edx -; AVX2-NEXT: movzwl %dx, %edx -; AVX2-NEXT: vmovd %edx, %xmm3 ; AVX2-NEXT: movq %rcx, %rdx ; AVX2-NEXT: sarq $63, %rdx -; AVX2-NEXT: vcvtph2ps %xmm3, %xmm3 +; AVX2-NEXT: vcvtph2ps %xmm0, %xmm3 ; AVX2-NEXT: andq %rax, %rdx ; AVX2-NEXT: vsubss %xmm1, %xmm3, %xmm4 ; AVX2-NEXT: vcvttss2si %xmm4, %rax @@ -3533,10 +3476,7 @@ define <4 x i32> @utesth_f16i32_mm(<4 x half> %x) nounwind { ; AVX2-NEXT: sarq $63, %rdx ; AVX2-NEXT: andq %rax, %rdx ; AVX2-NEXT: orq %rcx, %rdx -; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0 -; AVX2-NEXT: vpextrw $0, %xmm0, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm0 +; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] ; AVX2-NEXT: vcvtph2ps %xmm0, %xmm0 ; AVX2-NEXT: vsubss %xmm1, %xmm0, %xmm1 ; AVX2-NEXT: vcvttss2si %xmm1, %rax @@ -3668,31 +3608,19 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) nounwind { ; ; AVX2-LABEL: ustest_f16i32_mm: ; AVX2: # %bb.0: # %entry -; AVX2-NEXT: vpsrlq $48, %xmm0, %xmm1 -; AVX2-NEXT: vpextrw $0, %xmm1, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm1 +; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[3,3,3,3,4,5,6,7] ; AVX2-NEXT: vcvtph2ps %xmm1, %xmm1 ; AVX2-NEXT: vcvttss2si %xmm1, %rax ; AVX2-NEXT: vmovq %rax, %xmm1 ; AVX2-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3] -; AVX2-NEXT: vpextrw $0, %xmm2, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm2 ; AVX2-NEXT: vcvtph2ps %xmm2, %xmm2 ; AVX2-NEXT: vcvttss2si %xmm2, %rax ; AVX2-NEXT: vmovq %rax, %xmm2 ; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX2-NEXT: vpextrw $0, %xmm0, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm2 -; AVX2-NEXT: vcvtph2ps %xmm2, %xmm2 +; AVX2-NEXT: vcvtph2ps %xmm0, %xmm2 ; AVX2-NEXT: vcvttss2si %xmm2, %rax ; AVX2-NEXT: vmovq %rax, %xmm2 -; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0 -; AVX2-NEXT: vpextrw $0, %xmm0, %eax -; AVX2-NEXT: movzwl %ax, %eax -; AVX2-NEXT: vmovd %eax, %xmm0 +; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] ; AVX2-NEXT: vcvtph2ps %xmm0, %xmm0 ; AVX2-NEXT: vcvttss2si %xmm0, %rax ; AVX2-NEXT: vmovq %rax, %xmm0 diff --git a/llvm/test/CodeGen/X86/half.ll b/llvm/test/CodeGen/X86/half.ll index d0853fdc..9f01d07 100644 --- a/llvm/test/CodeGen/X86/half.ll +++ b/llvm/test/CodeGen/X86/half.ll @@ -851,16 +851,14 @@ define float @test_sitofp_fadd_i32(i32 %a, ptr %b) #0 { ; ; BWON-F16C-LABEL: test_sitofp_fadd_i32: ; BWON-F16C: # %bb.0: -; BWON-F16C-NEXT: movzwl (%rsi), %eax ; BWON-F16C-NEXT: vcvtsi2ss %edi, %xmm0, %xmm0 ; BWON-F16C-NEXT: vcvtps2ph $4, %xmm0, %xmm0 -; BWON-F16C-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero ; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm0 +; BWON-F16C-NEXT: movzwl (%rsi), %eax ; BWON-F16C-NEXT: vmovd %eax, %xmm1 ; BWON-F16C-NEXT: vcvtph2ps %xmm1, %xmm1 ; BWON-F16C-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; BWON-F16C-NEXT: vcvtps2ph $4, %xmm0, %xmm0 -; BWON-F16C-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero ; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm0 ; BWON-F16C-NEXT: retq ; @@ -919,7 +917,6 @@ define half @PR40273(half) #0 { ; BWON-F16C-LABEL: PR40273: ; BWON-F16C: # %bb.0: ; BWON-F16C-NEXT: vpextrw $0, %xmm0, %eax -; BWON-F16C-NEXT: movzwl %ax, %eax ; BWON-F16C-NEXT: vmovd %eax, %xmm0 ; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm0 ; BWON-F16C-NEXT: xorl %eax, %eax @@ -973,7 +970,6 @@ define void @brcond(half %0) #0 { ; BWON-F16C-LABEL: brcond: ; BWON-F16C: # %bb.0: # %entry ; BWON-F16C-NEXT: vpextrw $0, %xmm0, %eax -; BWON-F16C-NEXT: movzwl %ax, %eax ; BWON-F16C-NEXT: vmovd %eax, %xmm0 ; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm0 ; BWON-F16C-NEXT: vxorps %xmm1, %xmm1, %xmm1 @@ -1029,7 +1025,6 @@ define half @test_sqrt(half %0) #0 { ; BWON-F16C-LABEL: test_sqrt: ; BWON-F16C: # %bb.0: # %entry ; BWON-F16C-NEXT: vpextrw $0, %xmm0, %eax -; BWON-F16C-NEXT: movzwl %ax, %eax ; BWON-F16C-NEXT: vmovd %eax, %xmm0 ; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm0 ; BWON-F16C-NEXT: vsqrtss %xmm0, %xmm0, %xmm0 @@ -1083,7 +1078,6 @@ define void @main.158() #0 { ; BWON-F16C: # %bb.0: # %entry ; BWON-F16C-NEXT: vxorps %xmm0, %xmm0, %xmm0 ; BWON-F16C-NEXT: vcvtps2ph $4, %xmm0, %xmm1 -; BWON-F16C-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero ; BWON-F16C-NEXT: vcvtph2ps %xmm1, %xmm1 ; BWON-F16C-NEXT: vmovss {{.*#+}} xmm2 = [8.0E+0,0.0E+0,0.0E+0,0.0E+0] ; BWON-F16C-NEXT: vucomiss %xmm1, %xmm2 @@ -1172,8 +1166,7 @@ define void @main.45() #0 { ; ; BWON-F16C-LABEL: main.45: ; BWON-F16C: # %bb.0: # %entry -; BWON-F16C-NEXT: movzwl (%rax), %eax -; BWON-F16C-NEXT: vmovd %eax, %xmm0 +; BWON-F16C-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero ; BWON-F16C-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7] ; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm0 ; BWON-F16C-NEXT: xorl %eax, %eax @@ -1345,10 +1338,8 @@ define half @pr61271(half %0, half %1) #0 { ; BWON-F16C: # %bb.0: ; BWON-F16C-NEXT: vpextrw $0, %xmm0, %eax ; BWON-F16C-NEXT: vpextrw $0, %xmm1, %ecx -; BWON-F16C-NEXT: movzwl %cx, %ecx ; BWON-F16C-NEXT: vmovd %ecx, %xmm0 ; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm0 -; BWON-F16C-NEXT: movzwl %ax, %eax ; BWON-F16C-NEXT: vmovd %eax, %xmm1 ; BWON-F16C-NEXT: vcvtph2ps %xmm1, %xmm1 ; BWON-F16C-NEXT: vminss %xmm0, %xmm1, %xmm0 @@ -1615,14 +1606,8 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 { ; BWON-F16C-LABEL: maxnum_v8f16: ; BWON-F16C: # %bb.0: ; BWON-F16C-NEXT: vpsrldq {{.*#+}} xmm2 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; BWON-F16C-NEXT: vpextrw $0, %xmm2, %eax -; BWON-F16C-NEXT: movzwl %ax, %eax -; BWON-F16C-NEXT: vmovd %eax, %xmm2 ; BWON-F16C-NEXT: vcvtph2ps %xmm2, %xmm2 ; BWON-F16C-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; BWON-F16C-NEXT: vpextrw $0, %xmm3, %eax -; BWON-F16C-NEXT: movzwl %ax, %eax -; BWON-F16C-NEXT: vmovd %eax, %xmm3 ; BWON-F16C-NEXT: vcvtph2ps %xmm3, %xmm3 ; BWON-F16C-NEXT: vucomiss %xmm2, %xmm3 ; BWON-F16C-NEXT: ja .LBB26_2 @@ -1631,14 +1616,8 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 { ; BWON-F16C-NEXT: .LBB26_2: ; BWON-F16C-NEXT: vcvtps2ph $4, %xmm3, %xmm2 ; BWON-F16C-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[3,3,3,3] -; BWON-F16C-NEXT: vpextrw $0, %xmm3, %eax -; BWON-F16C-NEXT: movzwl %ax, %eax -; BWON-F16C-NEXT: vmovd %eax, %xmm3 ; BWON-F16C-NEXT: vcvtph2ps %xmm3, %xmm3 ; BWON-F16C-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[3,3,3,3] -; BWON-F16C-NEXT: vpextrw $0, %xmm4, %eax -; BWON-F16C-NEXT: movzwl %ax, %eax -; BWON-F16C-NEXT: vmovd %eax, %xmm4 ; BWON-F16C-NEXT: vcvtph2ps %xmm4, %xmm4 ; BWON-F16C-NEXT: vucomiss %xmm3, %xmm4 ; BWON-F16C-NEXT: ja .LBB26_4 @@ -1649,48 +1628,30 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 { ; BWON-F16C-NEXT: vcvtps2ph $4, %xmm4, %xmm2 ; BWON-F16C-NEXT: vmovd %xmm2, %ecx ; BWON-F16C-NEXT: vpsrldq {{.*#+}} xmm2 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; BWON-F16C-NEXT: vpextrw $0, %xmm2, %edx -; BWON-F16C-NEXT: movzwl %dx, %edx -; BWON-F16C-NEXT: vmovd %edx, %xmm2 +; BWON-F16C-NEXT: vcvtph2ps %xmm2, %xmm3 +; BWON-F16C-NEXT: vpsrldq {{.*#+}} xmm2 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero ; BWON-F16C-NEXT: vcvtph2ps %xmm2, %xmm2 -; BWON-F16C-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; BWON-F16C-NEXT: vpextrw $0, %xmm3, %edx -; BWON-F16C-NEXT: movzwl %dx, %edx -; BWON-F16C-NEXT: vmovd %edx, %xmm3 -; BWON-F16C-NEXT: vcvtph2ps %xmm3, %xmm3 -; BWON-F16C-NEXT: vucomiss %xmm2, %xmm3 +; BWON-F16C-NEXT: vucomiss %xmm3, %xmm2 ; BWON-F16C-NEXT: ja .LBB26_6 ; BWON-F16C-NEXT: # %bb.5: -; BWON-F16C-NEXT: vmovaps %xmm2, %xmm3 +; BWON-F16C-NEXT: vmovaps %xmm3, %xmm2 ; BWON-F16C-NEXT: .LBB26_6: -; BWON-F16C-NEXT: vcvtps2ph $4, %xmm3, %xmm2 +; BWON-F16C-NEXT: vcvtps2ph $4, %xmm2, %xmm2 ; BWON-F16C-NEXT: vmovd %xmm2, %edx ; BWON-F16C-NEXT: vshufpd {{.*#+}} xmm2 = xmm1[1,0] -; BWON-F16C-NEXT: vpextrw $0, %xmm2, %esi -; BWON-F16C-NEXT: movzwl %si, %esi -; BWON-F16C-NEXT: vmovd %esi, %xmm2 +; BWON-F16C-NEXT: vcvtph2ps %xmm2, %xmm3 +; BWON-F16C-NEXT: vshufpd {{.*#+}} xmm2 = xmm0[1,0] ; BWON-F16C-NEXT: vcvtph2ps %xmm2, %xmm2 -; BWON-F16C-NEXT: vshufpd {{.*#+}} xmm3 = xmm0[1,0] -; BWON-F16C-NEXT: vpextrw $0, %xmm3, %esi -; BWON-F16C-NEXT: movzwl %si, %esi -; BWON-F16C-NEXT: vmovd %esi, %xmm3 -; BWON-F16C-NEXT: vcvtph2ps %xmm3, %xmm3 -; BWON-F16C-NEXT: vucomiss %xmm2, %xmm3 +; BWON-F16C-NEXT: vucomiss %xmm3, %xmm2 ; BWON-F16C-NEXT: ja .LBB26_8 ; BWON-F16C-NEXT: # %bb.7: -; BWON-F16C-NEXT: vmovaps %xmm2, %xmm3 +; BWON-F16C-NEXT: vmovaps %xmm3, %xmm2 ; BWON-F16C-NEXT: .LBB26_8: -; BWON-F16C-NEXT: vcvtps2ph $4, %xmm3, %xmm2 +; BWON-F16C-NEXT: vcvtps2ph $4, %xmm2, %xmm2 ; BWON-F16C-NEXT: vmovd %xmm2, %esi -; BWON-F16C-NEXT: vpsrlq $48, %xmm1, %xmm2 -; BWON-F16C-NEXT: vpextrw $0, %xmm2, %edi -; BWON-F16C-NEXT: movzwl %di, %edi -; BWON-F16C-NEXT: vmovd %edi, %xmm2 +; BWON-F16C-NEXT: vpshuflw {{.*#+}} xmm2 = xmm1[3,3,3,3,4,5,6,7] ; BWON-F16C-NEXT: vcvtph2ps %xmm2, %xmm2 -; BWON-F16C-NEXT: vpsrlq $48, %xmm0, %xmm3 -; BWON-F16C-NEXT: vpextrw $0, %xmm3, %edi -; BWON-F16C-NEXT: movzwl %di, %edi -; BWON-F16C-NEXT: vmovd %edi, %xmm3 +; BWON-F16C-NEXT: vpshuflw {{.*#+}} xmm3 = xmm0[3,3,3,3,4,5,6,7] ; BWON-F16C-NEXT: vcvtph2ps %xmm3, %xmm6 ; BWON-F16C-NEXT: vucomiss %xmm2, %xmm6 ; BWON-F16C-NEXT: ja .LBB26_10 @@ -1704,53 +1665,35 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 { ; BWON-F16C-NEXT: vcvtps2ph $4, %xmm6, %xmm6 ; BWON-F16C-NEXT: vmovd %xmm6, %eax ; BWON-F16C-NEXT: vmovshdup {{.*#+}} xmm6 = xmm1[1,1,3,3] -; BWON-F16C-NEXT: vpextrw $0, %xmm6, %ecx -; BWON-F16C-NEXT: movzwl %cx, %ecx -; BWON-F16C-NEXT: vmovd %ecx, %xmm6 +; BWON-F16C-NEXT: vcvtph2ps %xmm6, %xmm7 +; BWON-F16C-NEXT: vmovshdup {{.*#+}} xmm6 = xmm0[1,1,3,3] ; BWON-F16C-NEXT: vcvtph2ps %xmm6, %xmm6 -; BWON-F16C-NEXT: vmovshdup {{.*#+}} xmm7 = xmm0[1,1,3,3] -; BWON-F16C-NEXT: vpextrw $0, %xmm7, %ecx -; BWON-F16C-NEXT: movzwl %cx, %ecx -; BWON-F16C-NEXT: vmovd %ecx, %xmm7 -; BWON-F16C-NEXT: vcvtph2ps %xmm7, %xmm7 -; BWON-F16C-NEXT: vucomiss %xmm6, %xmm7 +; BWON-F16C-NEXT: vucomiss %xmm7, %xmm6 ; BWON-F16C-NEXT: ja .LBB26_12 ; BWON-F16C-NEXT: # %bb.11: -; BWON-F16C-NEXT: vmovaps %xmm6, %xmm7 +; BWON-F16C-NEXT: vmovaps %xmm7, %xmm6 ; BWON-F16C-NEXT: .LBB26_12: ; BWON-F16C-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] ; BWON-F16C-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3] ; BWON-F16C-NEXT: vpinsrw $0, %eax, %xmm0, %xmm4 -; BWON-F16C-NEXT: vcvtps2ph $4, %xmm7, %xmm5 +; BWON-F16C-NEXT: vcvtps2ph $4, %xmm6, %xmm5 ; BWON-F16C-NEXT: vmovd %xmm5, %eax ; BWON-F16C-NEXT: vpinsrw $0, %eax, %xmm0, %xmm5 -; BWON-F16C-NEXT: vpextrw $0, %xmm1, %eax -; BWON-F16C-NEXT: movzwl %ax, %eax -; BWON-F16C-NEXT: vmovd %eax, %xmm6 -; BWON-F16C-NEXT: vcvtph2ps %xmm6, %xmm6 -; BWON-F16C-NEXT: vpextrw $0, %xmm0, %eax -; BWON-F16C-NEXT: movzwl %ax, %eax -; BWON-F16C-NEXT: vmovd %eax, %xmm7 -; BWON-F16C-NEXT: vcvtph2ps %xmm7, %xmm7 -; BWON-F16C-NEXT: vucomiss %xmm6, %xmm7 +; BWON-F16C-NEXT: vcvtph2ps %xmm1, %xmm7 +; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm6 +; BWON-F16C-NEXT: vucomiss %xmm7, %xmm6 ; BWON-F16C-NEXT: ja .LBB26_14 ; BWON-F16C-NEXT: # %bb.13: -; BWON-F16C-NEXT: vmovaps %xmm6, %xmm7 +; BWON-F16C-NEXT: vmovaps %xmm7, %xmm6 ; BWON-F16C-NEXT: .LBB26_14: ; BWON-F16C-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] ; BWON-F16C-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3] -; BWON-F16C-NEXT: vcvtps2ph $4, %xmm7, %xmm4 +; BWON-F16C-NEXT: vcvtps2ph $4, %xmm6, %xmm4 ; BWON-F16C-NEXT: vmovd %xmm4, %eax ; BWON-F16C-NEXT: vpinsrw $0, %eax, %xmm0, %xmm4 -; BWON-F16C-NEXT: vpsrld $16, %xmm1, %xmm1 -; BWON-F16C-NEXT: vpextrw $0, %xmm1, %eax -; BWON-F16C-NEXT: movzwl %ax, %eax -; BWON-F16C-NEXT: vmovd %eax, %xmm1 +; BWON-F16C-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,1,1,1,4,5,6,7] ; BWON-F16C-NEXT: vcvtph2ps %xmm1, %xmm1 -; BWON-F16C-NEXT: vpsrld $16, %xmm0, %xmm0 -; BWON-F16C-NEXT: vpextrw $0, %xmm0, %eax -; BWON-F16C-NEXT: movzwl %ax, %eax -; BWON-F16C-NEXT: vmovd %eax, %xmm0 +; BWON-F16C-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] ; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm0 ; BWON-F16C-NEXT: vucomiss %xmm1, %xmm0 ; BWON-F16C-NEXT: ja .LBB26_16 diff --git a/llvm/test/CodeGen/X86/inline-asm-memop.ll b/llvm/test/CodeGen/X86/inline-asm-memop.ll new file mode 100644 index 0000000..83442498 --- /dev/null +++ b/llvm/test/CodeGen/X86/inline-asm-memop.ll @@ -0,0 +1,27 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O0 < %s | FileCheck %s + +; A bug in X86DAGToDAGISel::matchAddressRecursively create a zext SDValue which +; is quickly replaced by other SDValue but already pushed into vector for later +; calling for SelectionDAGISel::Select_INLINEASM getNode builder, see issue +; 82431 for more infomation. + +define void @PR82431(i8 %call, ptr %b) { +; CHECK-LABEL: PR82431: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movb %dil, %al +; CHECK-NEXT: addb $1, %al +; CHECK-NEXT: movzbl %al, %eax +; CHECK-NEXT: # kill: def $rax killed $eax +; CHECK-NEXT: shlq $3, %rax +; CHECK-NEXT: addq %rax, %rsi +; CHECK-NEXT: #APP +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: retq +entry: + %narrow = add nuw i8 %call, 1 + %idxprom = zext i8 %narrow to i64 + %arrayidx = getelementptr [1 x i64], ptr %b, i64 0, i64 %idxprom + tail call void asm "", "=*m,*m,~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %arrayidx, ptr elementtype(i64) %arrayidx) + ret void +} diff --git a/llvm/test/CodeGen/X86/pr31088.ll b/llvm/test/CodeGen/X86/pr31088.ll index fa1014e..ce37622 100644 --- a/llvm/test/CodeGen/X86/pr31088.ll +++ b/llvm/test/CodeGen/X86/pr31088.ll @@ -41,15 +41,9 @@ define <1 x half> @ir_fadd_v1f16(<1 x half> %arg0, <1 x half> %arg1) nounwind { ; ; F16C-LABEL: ir_fadd_v1f16: ; F16C: # %bb.0: -; F16C-NEXT: vpextrw $0, %xmm0, %eax -; F16C-NEXT: vpextrw $0, %xmm1, %ecx -; F16C-NEXT: movzwl %cx, %ecx -; F16C-NEXT: vmovd %ecx, %xmm0 -; F16C-NEXT: vcvtph2ps %xmm0, %xmm0 -; F16C-NEXT: movzwl %ax, %eax -; F16C-NEXT: vmovd %eax, %xmm1 ; F16C-NEXT: vcvtph2ps %xmm1, %xmm1 -; F16C-NEXT: vaddss %xmm0, %xmm1, %xmm0 +; F16C-NEXT: vcvtph2ps %xmm0, %xmm0 +; F16C-NEXT: vaddss %xmm1, %xmm0, %xmm0 ; F16C-NEXT: vcvtps2ph $4, %xmm0, %xmm0 ; F16C-NEXT: vmovd %xmm0, %eax ; F16C-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0 @@ -58,13 +52,15 @@ define <1 x half> @ir_fadd_v1f16(<1 x half> %arg0, <1 x half> %arg1) nounwind { ; F16C-O0-LABEL: ir_fadd_v1f16: ; F16C-O0: # %bb.0: ; F16C-O0-NEXT: vpextrw $0, %xmm1, %eax -; F16C-O0-NEXT: # kill: def $ax killed $ax killed $eax -; F16C-O0-NEXT: movzwl %ax, %eax +; F16C-O0-NEXT: movw %ax, %cx +; F16C-O0-NEXT: # implicit-def: $eax +; F16C-O0-NEXT: movw %cx, %ax ; F16C-O0-NEXT: vmovd %eax, %xmm1 ; F16C-O0-NEXT: vcvtph2ps %xmm1, %xmm1 ; F16C-O0-NEXT: vpextrw $0, %xmm0, %eax -; F16C-O0-NEXT: # kill: def $ax killed $ax killed $eax -; F16C-O0-NEXT: movzwl %ax, %eax +; F16C-O0-NEXT: movw %ax, %cx +; F16C-O0-NEXT: # implicit-def: $eax +; F16C-O0-NEXT: movw %cx, %ax ; F16C-O0-NEXT: vmovd %eax, %xmm0 ; F16C-O0-NEXT: vcvtph2ps %xmm0, %xmm0 ; F16C-O0-NEXT: vaddss %xmm1, %xmm0, %xmm0 diff --git a/llvm/test/CodeGen/X86/pr34605.ll b/llvm/test/CodeGen/X86/pr34605.ll index 863b0ff..25dd6a7 100644 --- a/llvm/test/CodeGen/X86/pr34605.ll +++ b/llvm/test/CodeGen/X86/pr34605.ll @@ -17,7 +17,7 @@ define void @pr34605(ptr nocapture %s, i32 %p) { ; CHECK-NEXT: kmovd %ecx, %k1 ; CHECK-NEXT: kmovd %k1, %k1 ; CHECK-NEXT: kandq %k1, %k0, %k1 -; CHECK-NEXT: vmovdqu8 {{\.?LCPI[0-9]+_[0-9]+}}, %zmm0 {%k1} {z} +; CHECK-NEXT: vmovdqu8 {{.*#+}} zmm0 {%k1} {z} = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ; CHECK-NEXT: vmovdqu64 %zmm0, (%eax) ; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0 ; CHECK-NEXT: vmovdqu64 %zmm0, 64(%eax) diff --git a/llvm/test/CodeGen/X86/pr38803.ll b/llvm/test/CodeGen/X86/pr38803.ll index 61dc228..ebac812 100644 --- a/llvm/test/CodeGen/X86/pr38803.ll +++ b/llvm/test/CodeGen/X86/pr38803.ll @@ -13,7 +13,7 @@ define dso_local float @_Z3fn2v() { ; CHECK-NEXT: callq _Z1av@PLT ; CHECK-NEXT: # kill: def $al killed $al def $eax ; CHECK-NEXT: kmovd %eax, %k1 -; CHECK-NEXT: vmovss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 {%k1} {z} +; CHECK-NEXT: vmovss {{.*#+}} xmm0 {%k1} {z} = [7.5E-1,0.0E+0,0.0E+0,0.0E+0] ; CHECK-NEXT: cmpl $0, c(%rip) ; CHECK-NEXT: je .LBB0_2 ; CHECK-NEXT: # %bb.1: # %if.then diff --git a/llvm/test/CodeGen/X86/pr43509.ll b/llvm/test/CodeGen/X86/pr43509.ll index 87ddad0..a29fe4c 100644 --- a/llvm/test/CodeGen/X86/pr43509.ll +++ b/llvm/test/CodeGen/X86/pr43509.ll @@ -7,7 +7,7 @@ define <8 x i8> @foo(<8 x float> %arg) { ; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1 ; CHECK-NEXT: vcmpltps %ymm1, %ymm0, %k1 ; CHECK-NEXT: vcmpgtps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %k1 {%k1} -; CHECK-NEXT: vmovdqu8 {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 {%k1} {z} +; CHECK-NEXT: vmovdqu8 {{.*#+}} xmm0 {%k1} {z} = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq bb: diff --git a/llvm/test/CodeGen/X86/pr57340.ll b/llvm/test/CodeGen/X86/pr57340.ll index 57f52c8..00a52c6 100644 --- a/llvm/test/CodeGen/X86/pr57340.ll +++ b/llvm/test/CodeGen/X86/pr57340.ll @@ -5,54 +5,42 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-LABEL: main.41: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vpbroadcastw (%rax), %xmm0 -; CHECK-NEXT: vmovdqu (%rax), %ymm2 -; CHECK-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm3 -; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm1 = [31,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14] -; CHECK-NEXT: vpermi2w %ymm3, %ymm2, %ymm1 ; CHECK-NEXT: vpextrw $0, %xmm0, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm0 -; CHECK-NEXT: vcvtph2ps %xmm0, %xmm0 +; CHECK-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm1 +; CHECK-NEXT: vmovdqu (%rax), %ymm3 +; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm2 = [31,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14] +; CHECK-NEXT: vpermi2w %ymm1, %ymm3, %ymm2 +; CHECK-NEXT: vprold $16, %xmm2, %xmm1 +; CHECK-NEXT: vcvtph2ps %xmm1, %xmm3 ; CHECK-NEXT: vmovdqu (%rax), %xmm5 -; CHECK-NEXT: vpextrw $0, %xmm5, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm2 -; CHECK-NEXT: vcvtph2ps %xmm2, %xmm2 -; CHECK-NEXT: vucomiss %xmm0, %xmm2 -; CHECK-NEXT: setnp %al -; CHECK-NEXT: sete %cl -; CHECK-NEXT: testb %al, %cl -; CHECK-NEXT: vpsrld $16, %xmm1, %xmm3 -; CHECK-NEXT: vpextrw $0, %xmm3, %eax -; CHECK-NEXT: movzwl %ax, %eax +; CHECK-NEXT: vprold $16, %xmm5, %xmm1 +; CHECK-NEXT: vcvtph2ps %xmm1, %xmm1 +; CHECK-NEXT: vucomiss %xmm3, %xmm1 +; CHECK-NEXT: setnp %cl +; CHECK-NEXT: sete %dl +; CHECK-NEXT: testb %cl, %dl +; CHECK-NEXT: setne %cl +; CHECK-NEXT: kmovd %ecx, %k0 +; CHECK-NEXT: kshiftlw $15, %k0, %k0 ; CHECK-NEXT: vmovd %eax, %xmm3 -; CHECK-NEXT: vpsrld $16, %xmm5, %xmm4 -; CHECK-NEXT: vpextrw $0, %xmm4, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm4 -; CHECK-NEXT: setne %al -; CHECK-NEXT: andl $1, %eax -; CHECK-NEXT: vcvtph2ps %xmm3, %xmm6 -; CHECK-NEXT: vcvtph2ps %xmm4, %xmm3 -; CHECK-NEXT: kmovw %eax, %k0 -; CHECK-NEXT: vucomiss %xmm6, %xmm3 +; CHECK-NEXT: vcvtph2ps %xmm3, %xmm3 +; CHECK-NEXT: vcvtph2ps %xmm5, %xmm6 +; CHECK-NEXT: kshiftrw $14, %k0, %k0 +; CHECK-NEXT: vucomiss %xmm3, %xmm6 ; CHECK-NEXT: setnp %al ; CHECK-NEXT: sete %cl ; CHECK-NEXT: testb %al, %cl ; CHECK-NEXT: setne %al -; CHECK-NEXT: kmovd %eax, %k1 -; CHECK-NEXT: kshiftlw $15, %k1, %k1 -; CHECK-NEXT: kshiftrw $14, %k1, %k1 -; CHECK-NEXT: korw %k1, %k0, %k0 +; CHECK-NEXT: andl $1, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: korw %k0, %k1, %k0 ; CHECK-NEXT: movw $-5, %ax ; CHECK-NEXT: kmovd %eax, %k1 ; CHECK-NEXT: kandw %k1, %k0, %k0 -; CHECK-NEXT: vprolq $32, %xmm1, %xmm4 -; CHECK-NEXT: vpextrw $0, %xmm4, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm4 -; CHECK-NEXT: vcvtph2ps %xmm4, %xmm4 -; CHECK-NEXT: vucomiss %xmm4, %xmm0 +; CHECK-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3] +; CHECK-NEXT: vcvtph2ps %xmm3, %xmm3 +; CHECK-NEXT: vcvtph2ps %xmm0, %xmm0 +; CHECK-NEXT: vucomiss %xmm3, %xmm0 ; CHECK-NEXT: setnp %al ; CHECK-NEXT: sete %cl ; CHECK-NEXT: testb %al, %cl @@ -63,18 +51,12 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-NEXT: korw %k1, %k0, %k0 ; CHECK-NEXT: movw $-9, %ax ; CHECK-NEXT: kmovd %eax, %k1 -; CHECK-NEXT: vpsrlq $48, %xmm1, %xmm4 -; CHECK-NEXT: vpextrw $0, %xmm4, %eax ; CHECK-NEXT: kandw %k1, %k0, %k0 -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm4 -; CHECK-NEXT: vcvtph2ps %xmm4, %xmm6 -; CHECK-NEXT: vpsrlq $48, %xmm5, %xmm4 -; CHECK-NEXT: vpextrw $0, %xmm4, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm4 -; CHECK-NEXT: vcvtph2ps %xmm4, %xmm4 -; CHECK-NEXT: vucomiss %xmm6, %xmm4 +; CHECK-NEXT: vprolq $16, %xmm2, %xmm3 +; CHECK-NEXT: vcvtph2ps %xmm3, %xmm4 +; CHECK-NEXT: vprolq $16, %xmm5, %xmm3 +; CHECK-NEXT: vcvtph2ps %xmm3, %xmm3 +; CHECK-NEXT: vucomiss %xmm4, %xmm3 ; CHECK-NEXT: setnp %al ; CHECK-NEXT: sete %cl ; CHECK-NEXT: testb %al, %cl @@ -85,13 +67,10 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-NEXT: korw %k1, %k0, %k0 ; CHECK-NEXT: movw $-17, %ax ; CHECK-NEXT: kmovd %eax, %k1 -; CHECK-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[2,3,0,1] -; CHECK-NEXT: vpextrw $0, %xmm6, %eax ; CHECK-NEXT: kandw %k1, %k0, %k0 -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm6 -; CHECK-NEXT: vcvtph2ps %xmm6, %xmm6 -; CHECK-NEXT: vucomiss %xmm6, %xmm0 +; CHECK-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,0,1] +; CHECK-NEXT: vcvtph2ps %xmm4, %xmm4 +; CHECK-NEXT: vucomiss %xmm4, %xmm0 ; CHECK-NEXT: setnp %al ; CHECK-NEXT: sete %cl ; CHECK-NEXT: testb %al, %cl @@ -102,18 +81,12 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-NEXT: korw %k1, %k0, %k0 ; CHECK-NEXT: movw $-33, %ax ; CHECK-NEXT: kmovd %eax, %k1 -; CHECK-NEXT: vpsrldq {{.*#+}} xmm6 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; CHECK-NEXT: vpextrw $0, %xmm6, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm6 -; CHECK-NEXT: vcvtph2ps %xmm6, %xmm7 -; CHECK-NEXT: vpsrldq {{.*#+}} xmm6 = xmm5[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; CHECK-NEXT: vpextrw $0, %xmm6, %eax +; CHECK-NEXT: vpsrldq {{.*#+}} xmm4 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; CHECK-NEXT: vcvtph2ps %xmm4, %xmm7 +; CHECK-NEXT: vpsrldq {{.*#+}} xmm4 = xmm5[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; CHECK-NEXT: vcvtph2ps %xmm4, %xmm4 ; CHECK-NEXT: kandw %k1, %k0, %k0 -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm6 -; CHECK-NEXT: vcvtph2ps %xmm6, %xmm6 -; CHECK-NEXT: vucomiss %xmm7, %xmm6 +; CHECK-NEXT: vucomiss %xmm7, %xmm4 ; CHECK-NEXT: setnp %al ; CHECK-NEXT: sete %cl ; CHECK-NEXT: testb %al, %cl @@ -125,10 +98,7 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-NEXT: movw $-65, %ax ; CHECK-NEXT: kmovd %eax, %k1 ; CHECK-NEXT: kandw %k1, %k0, %k0 -; CHECK-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[3,3,3,3] -; CHECK-NEXT: vpextrw $0, %xmm7, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm7 +; CHECK-NEXT: vshufps {{.*#+}} xmm7 = xmm2[3,3,3,3] ; CHECK-NEXT: vcvtph2ps %xmm7, %xmm7 ; CHECK-NEXT: vucomiss %xmm7, %xmm0 ; CHECK-NEXT: setnp %al @@ -142,15 +112,9 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-NEXT: movw $-129, %ax ; CHECK-NEXT: kmovd %eax, %k1 ; CHECK-NEXT: kandw %k1, %k0, %k0 -; CHECK-NEXT: vpsrldq {{.*#+}} xmm7 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; CHECK-NEXT: vpextrw $0, %xmm7, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm7 +; CHECK-NEXT: vpsrldq {{.*#+}} xmm7 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero ; CHECK-NEXT: vcvtph2ps %xmm7, %xmm7 ; CHECK-NEXT: vpsrldq {{.*#+}} xmm5 = xmm5[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; CHECK-NEXT: vpextrw $0, %xmm5, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm5 ; CHECK-NEXT: vcvtph2ps %xmm5, %xmm5 ; CHECK-NEXT: vucomiss %xmm7, %xmm5 ; CHECK-NEXT: setnp %al @@ -163,13 +127,10 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-NEXT: korw %k1, %k0, %k0 ; CHECK-NEXT: movw $-257, %ax # imm = 0xFEFF ; CHECK-NEXT: kmovd %eax, %k1 +; CHECK-NEXT: vextracti128 $1, %ymm2, %xmm2 +; CHECK-NEXT: vcvtph2ps %xmm2, %xmm7 ; CHECK-NEXT: kandw %k1, %k0, %k0 -; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm1 -; CHECK-NEXT: vpextrw $0, %xmm1, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm7 -; CHECK-NEXT: vcvtph2ps %xmm7, %xmm7 -; CHECK-NEXT: vucomiss %xmm7, %xmm2 +; CHECK-NEXT: vucomiss %xmm7, %xmm6 ; CHECK-NEXT: setnp %al ; CHECK-NEXT: sete %cl ; CHECK-NEXT: testb %al, %cl @@ -181,12 +142,9 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-NEXT: movw $-513, %ax # imm = 0xFDFF ; CHECK-NEXT: kmovd %eax, %k1 ; CHECK-NEXT: kandw %k1, %k0, %k0 -; CHECK-NEXT: vpsrld $16, %xmm1, %xmm2 -; CHECK-NEXT: vpextrw $0, %xmm2, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm2 -; CHECK-NEXT: vcvtph2ps %xmm2, %xmm2 -; CHECK-NEXT: vucomiss %xmm2, %xmm3 +; CHECK-NEXT: vprold $16, %xmm2, %xmm6 +; CHECK-NEXT: vcvtph2ps %xmm6, %xmm6 +; CHECK-NEXT: vucomiss %xmm6, %xmm1 ; CHECK-NEXT: setnp %al ; CHECK-NEXT: sete %cl ; CHECK-NEXT: testb %al, %cl @@ -197,13 +155,10 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-NEXT: korw %k1, %k0, %k0 ; CHECK-NEXT: movw $-1025, %ax # imm = 0xFBFF ; CHECK-NEXT: kmovd %eax, %k1 -; CHECK-NEXT: vprolq $32, %xmm1, %xmm2 -; CHECK-NEXT: vpextrw $0, %xmm2, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm2 -; CHECK-NEXT: vcvtph2ps %xmm2, %xmm2 ; CHECK-NEXT: kandw %k1, %k0, %k0 -; CHECK-NEXT: vucomiss %xmm2, %xmm0 +; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[1,1,3,3] +; CHECK-NEXT: vcvtph2ps %xmm1, %xmm1 +; CHECK-NEXT: vucomiss %xmm1, %xmm0 ; CHECK-NEXT: setnp %al ; CHECK-NEXT: sete %cl ; CHECK-NEXT: testb %al, %cl @@ -215,12 +170,9 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-NEXT: movw $-2049, %ax # imm = 0xF7FF ; CHECK-NEXT: kmovd %eax, %k1 ; CHECK-NEXT: kandw %k1, %k0, %k0 -; CHECK-NEXT: vpsrlq $48, %xmm1, %xmm2 -; CHECK-NEXT: vpextrw $0, %xmm2, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm2 -; CHECK-NEXT: vcvtph2ps %xmm2, %xmm2 -; CHECK-NEXT: vucomiss %xmm2, %xmm4 +; CHECK-NEXT: vprolq $16, %xmm2, %xmm1 +; CHECK-NEXT: vcvtph2ps %xmm1, %xmm1 +; CHECK-NEXT: vucomiss %xmm1, %xmm3 ; CHECK-NEXT: setnp %al ; CHECK-NEXT: sete %cl ; CHECK-NEXT: testb %al, %cl @@ -231,13 +183,10 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-NEXT: korw %k1, %k0, %k0 ; CHECK-NEXT: movw $-4097, %ax # imm = 0xEFFF ; CHECK-NEXT: kmovd %eax, %k1 -; CHECK-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1] -; CHECK-NEXT: vpextrw $0, %xmm2, %eax ; CHECK-NEXT: kandw %k1, %k0, %k0 -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm2 -; CHECK-NEXT: vcvtph2ps %xmm2, %xmm2 -; CHECK-NEXT: vucomiss %xmm2, %xmm0 +; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,0,1] +; CHECK-NEXT: vcvtph2ps %xmm1, %xmm1 +; CHECK-NEXT: vucomiss %xmm1, %xmm0 ; CHECK-NEXT: setnp %al ; CHECK-NEXT: sete %cl ; CHECK-NEXT: testb %al, %cl @@ -249,12 +198,9 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-NEXT: movw $-8193, %ax # imm = 0xDFFF ; CHECK-NEXT: kmovd %eax, %k1 ; CHECK-NEXT: kandw %k1, %k0, %k0 -; CHECK-NEXT: vpsrldq {{.*#+}} xmm2 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; CHECK-NEXT: vpextrw $0, %xmm2, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm2 -; CHECK-NEXT: vcvtph2ps %xmm2, %xmm2 -; CHECK-NEXT: vucomiss %xmm2, %xmm6 +; CHECK-NEXT: vpsrldq {{.*#+}} xmm1 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; CHECK-NEXT: vcvtph2ps %xmm1, %xmm1 +; CHECK-NEXT: vucomiss %xmm1, %xmm4 ; CHECK-NEXT: setnp %al ; CHECK-NEXT: sete %cl ; CHECK-NEXT: testb %al, %cl @@ -265,13 +211,10 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-NEXT: korw %k1, %k0, %k0 ; CHECK-NEXT: movw $-16385, %ax # imm = 0xBFFF ; CHECK-NEXT: kmovd %eax, %k1 +; CHECK-NEXT: vshufps {{.*#+}} xmm1 = xmm2[3,3,3,3] +; CHECK-NEXT: vcvtph2ps %xmm1, %xmm1 ; CHECK-NEXT: kandw %k1, %k0, %k0 -; CHECK-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[3,3,3,3] -; CHECK-NEXT: vpextrw $0, %xmm2, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm2 -; CHECK-NEXT: vcvtph2ps %xmm2, %xmm2 -; CHECK-NEXT: vucomiss %xmm2, %xmm0 +; CHECK-NEXT: vucomiss %xmm1, %xmm0 ; CHECK-NEXT: setnp %al ; CHECK-NEXT: sete %cl ; CHECK-NEXT: testb %al, %cl @@ -280,10 +223,7 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-NEXT: kshiftlw $14, %k1, %k1 ; CHECK-NEXT: korw %k1, %k0, %k0 ; CHECK-NEXT: kshiftlw $1, %k0, %k0 -; CHECK-NEXT: vpsrldq {{.*#+}} xmm0 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; CHECK-NEXT: vpextrw $0, %xmm0, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: vmovd %eax, %xmm0 +; CHECK-NEXT: vpsrldq {{.*#+}} xmm0 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero ; CHECK-NEXT: vcvtph2ps %xmm0, %xmm0 ; CHECK-NEXT: kshiftrw $1, %k0, %k0 ; CHECK-NEXT: vucomiss %xmm0, %xmm5 @@ -294,7 +234,7 @@ define void @main.41() local_unnamed_addr #1 { ; CHECK-NEXT: kmovd %eax, %k1 ; CHECK-NEXT: kshiftlw $15, %k1, %k1 ; CHECK-NEXT: korw %k1, %k0, %k1 -; CHECK-NEXT: vmovdqu8 {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 {%k1} {z} +; CHECK-NEXT: vmovdqu8 {{.*#+}} xmm0 {%k1} {z} = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ; CHECK-NEXT: vmovdqa %xmm0, (%rax) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq diff --git a/llvm/test/CodeGen/X86/pr78897.ll b/llvm/test/CodeGen/X86/pr78897.ll index 0c1c3ca..56e4ec2 100644 --- a/llvm/test/CodeGen/X86/pr78897.ll +++ b/llvm/test/CodeGen/X86/pr78897.ll @@ -225,7 +225,7 @@ define <16 x i8> @produceShuffleVectorForByte(i8 zeroext %0) nounwind { ; X86-AVX512-NEXT: pushl %esi ; X86-AVX512-NEXT: vpbroadcastb {{[0-9]+}}(%esp), %xmm0 ; X86-AVX512-NEXT: vptestnmb {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %k1 -; X86-AVX512-NEXT: vmovdqu8 {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 {%k1} {z} +; X86-AVX512-NEXT: vmovdqu8 {{.*#+}} xmm0 {%k1} {z} = [17,17,17,17,17,17,17,17,u,u,u,u,u,u,u,u] ; X86-AVX512-NEXT: vpextrd $1, %xmm0, %eax ; X86-AVX512-NEXT: vmovd %xmm0, %edx ; X86-AVX512-NEXT: movl $286331152, %ecx # imm = 0x11111110 @@ -258,7 +258,7 @@ define <16 x i8> @produceShuffleVectorForByte(i8 zeroext %0) nounwind { ; X64-AVX512: # %bb.0: # %entry ; X64-AVX512-NEXT: vpbroadcastb %edi, %xmm0 ; X64-AVX512-NEXT: vptestnmb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %k1 -; X64-AVX512-NEXT: vmovdqu8 {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 {%k1} {z} +; X64-AVX512-NEXT: vmovdqu8 {{.*#+}} xmm0 {%k1} {z} = [17,17,17,17,17,17,17,17,u,u,u,u,u,u,u,u] ; X64-AVX512-NEXT: vmovq %xmm0, %rax ; X64-AVX512-NEXT: movabsq $1229782938247303440, %rcx # imm = 0x1111111111111110 ; X64-AVX512-NEXT: movabsq $76861433640456465, %rdx # imm = 0x111111111111111 diff --git a/llvm/test/CodeGen/X86/prefer-fpext-splat.ll b/llvm/test/CodeGen/X86/prefer-fpext-splat.ll index 1d8b8b3..c3d7b2e 100644 --- a/llvm/test/CodeGen/X86/prefer-fpext-splat.ll +++ b/llvm/test/CodeGen/X86/prefer-fpext-splat.ll @@ -176,8 +176,6 @@ define <2 x double> @prefer_f16_v2f64(ptr %p) nounwind { ; AVX512F-LABEL: prefer_f16_v2f64: ; AVX512F: # %bb.0: # %entry ; AVX512F-NEXT: vpbroadcastw (%rdi), %xmm0 -; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX512F-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3] ; AVX512F-NEXT: vcvtph2ps %xmm0, %xmm0 ; AVX512F-NEXT: vcvtps2pd %xmm0, %xmm0 ; AVX512F-NEXT: retq diff --git a/llvm/test/CodeGen/X86/select-of-fp-constants.ll b/llvm/test/CodeGen/X86/select-of-fp-constants.ll index 76b8ea8..2cdaa11 100644 --- a/llvm/test/CodeGen/X86/select-of-fp-constants.ll +++ b/llvm/test/CodeGen/X86/select-of-fp-constants.ll @@ -86,7 +86,7 @@ define float @fcmp_select_fp_constants(float %x) nounwind readnone { ; X64-AVX512F: # %bb.0: ; X64-AVX512F-NEXT: vcmpneqss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %k1 ; X64-AVX512F-NEXT: vmovss {{.*#+}} xmm0 = [2.3E+1,0.0E+0,0.0E+0,0.0E+0] -; X64-AVX512F-NEXT: vmovss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 {%k1} +; X64-AVX512F-NEXT: vmovss {{.*#+}} xmm0 {%k1} = [4.2E+1,0.0E+0,0.0E+0,0.0E+0] ; X64-AVX512F-NEXT: retq %c = fcmp une float %x, -4.0 %r = select i1 %c, float 42.0, float 23.0 diff --git a/llvm/test/CodeGen/X86/select-of-half-constants.ll b/llvm/test/CodeGen/X86/select-of-half-constants.ll index e22d4c8..e3d92eb 100644 --- a/llvm/test/CodeGen/X86/select-of-half-constants.ll +++ b/llvm/test/CodeGen/X86/select-of-half-constants.ll @@ -6,9 +6,9 @@ define half @fcmp_select_fp_constants_olt(half %x) nounwind readnone { ; X64-AVX512FP16-LABEL: fcmp_select_fp_constants_olt: ; X64-AVX512FP16: # %bb.0: -; X64-AVX512FP16-NEXT: vmovsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; X64-AVX512FP16-NEXT: vmovsh {{.*#+}} xmm1 = [4.2E+1,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0] ; X64-AVX512FP16-NEXT: vcmpltsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %k1 -; X64-AVX512FP16-NEXT: vmovsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-AVX512FP16-NEXT: vmovsh {{.*#+}} xmm0 = [2.3E+1,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0] ; X64-AVX512FP16-NEXT: vmovsh %xmm1, %xmm0, %xmm0 {%k1} ; X64-AVX512FP16-NEXT: retq %c = fcmp olt half %x, -4.0 @@ -19,9 +19,9 @@ define half @fcmp_select_fp_constants_olt(half %x) nounwind readnone { define half @fcmp_select_fp_constants_ogt(half %x) nounwind readnone { ; X64-AVX512FP16-LABEL: fcmp_select_fp_constants_ogt: ; X64-AVX512FP16: # %bb.0: -; X64-AVX512FP16-NEXT: vmovsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; X64-AVX512FP16-NEXT: vmovsh {{.*#+}} xmm1 = [4.2E+1,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0] ; X64-AVX512FP16-NEXT: vcmpgtsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %k1 -; X64-AVX512FP16-NEXT: vmovsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-AVX512FP16-NEXT: vmovsh {{.*#+}} xmm0 = [2.3E+1,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0] ; X64-AVX512FP16-NEXT: vmovsh %xmm1, %xmm0, %xmm0 {%k1} ; X64-AVX512FP16-NEXT: retq %c = fcmp ogt half %x, -4.0 diff --git a/llvm/test/CodeGen/X86/vector-half-conversions.ll b/llvm/test/CodeGen/X86/vector-half-conversions.ll index f59960f..ba21af2 100644 --- a/llvm/test/CodeGen/X86/vector-half-conversions.ll +++ b/llvm/test/CodeGen/X86/vector-half-conversions.ll @@ -21,15 +21,13 @@ define float @cvt_i16_to_f32(i16 %a0) nounwind { ; ; F16C-LABEL: cvt_i16_to_f32: ; F16C: # %bb.0: -; F16C-NEXT: movzwl %di, %eax -; F16C-NEXT: vmovd %eax, %xmm0 +; F16C-NEXT: vmovd %edi, %xmm0 ; F16C-NEXT: vcvtph2ps %xmm0, %xmm0 ; F16C-NEXT: retq ; ; AVX512-LABEL: cvt_i16_to_f32: ; AVX512: # %bb.0: -; AVX512-NEXT: movzwl %di, %eax -; AVX512-NEXT: vmovd %eax, %xmm0 +; AVX512-NEXT: vmovd %edi, %xmm0 ; AVX512-NEXT: vcvtph2ps %xmm0, %xmm0 ; AVX512-NEXT: retq %1 = bitcast i16 %a0 to half @@ -1370,16 +1368,14 @@ define double @cvt_i16_to_f64(i16 %a0) nounwind { ; ; F16C-LABEL: cvt_i16_to_f64: ; F16C: # %bb.0: -; F16C-NEXT: movzwl %di, %eax -; F16C-NEXT: vmovd %eax, %xmm0 +; F16C-NEXT: vmovd %edi, %xmm0 ; F16C-NEXT: vcvtph2ps %xmm0, %xmm0 ; F16C-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0 ; F16C-NEXT: retq ; ; AVX512-LABEL: cvt_i16_to_f64: ; AVX512: # %bb.0: -; AVX512-NEXT: movzwl %di, %eax -; AVX512-NEXT: vmovd %eax, %xmm0 +; AVX512-NEXT: vmovd %edi, %xmm0 ; AVX512-NEXT: vcvtph2ps %xmm0, %xmm0 ; AVX512-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0 ; AVX512-NEXT: retq @@ -1410,14 +1406,12 @@ define <2 x double> @cvt_2i16_to_2f64(<2 x i16> %a0) nounwind { ; ; F16C-LABEL: cvt_2i16_to_2f64: ; F16C: # %bb.0: -; F16C-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero ; F16C-NEXT: vcvtph2ps %xmm0, %xmm0 ; F16C-NEXT: vcvtps2pd %xmm0, %xmm0 ; F16C-NEXT: retq ; ; AVX512-LABEL: cvt_2i16_to_2f64: ; AVX512: # %bb.0: -; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero ; AVX512-NEXT: vcvtph2ps %xmm0, %xmm0 ; AVX512-NEXT: vcvtps2pd %xmm0, %xmm0 ; AVX512-NEXT: retq @@ -1503,14 +1497,12 @@ define <2 x double> @cvt_8i16_to_2f64(<8 x i16> %a0) nounwind { ; ; F16C-LABEL: cvt_8i16_to_2f64: ; F16C: # %bb.0: -; F16C-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero ; F16C-NEXT: vcvtph2ps %xmm0, %xmm0 ; F16C-NEXT: vcvtps2pd %xmm0, %xmm0 ; F16C-NEXT: retq ; ; AVX512-LABEL: cvt_8i16_to_2f64: ; AVX512: # %bb.0: -; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero ; AVX512-NEXT: vcvtph2ps %xmm0, %xmm0 ; AVX512-NEXT: vcvtps2pd %xmm0, %xmm0 ; AVX512-NEXT: retq @@ -1877,16 +1869,14 @@ define <2 x double> @load_cvt_2i16_to_2f64(ptr %a0) nounwind { ; ; F16C-LABEL: load_cvt_2i16_to_2f64: ; F16C: # %bb.0: -; F16C-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; F16C-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero +; F16C-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero ; F16C-NEXT: vcvtph2ps %xmm0, %xmm0 ; F16C-NEXT: vcvtps2pd %xmm0, %xmm0 ; F16C-NEXT: retq ; ; AVX512-LABEL: load_cvt_2i16_to_2f64: ; AVX512: # %bb.0: -; AVX512-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero +; AVX512-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero ; AVX512-NEXT: vcvtph2ps %xmm0, %xmm0 ; AVX512-NEXT: vcvtps2pd %xmm0, %xmm0 ; AVX512-NEXT: retq @@ -4976,32 +4966,22 @@ define <4 x i32> @fptosi_2f16_to_4i32(<2 x half> %a) nounwind { ; ; F16C-LABEL: fptosi_2f16_to_4i32: ; F16C: # %bb.0: -; F16C-NEXT: vpextrw $0, %xmm0, %eax -; F16C-NEXT: movzwl %ax, %eax -; F16C-NEXT: vmovd %eax, %xmm1 +; F16C-NEXT: vpsrld $16, %xmm0, %xmm1 ; F16C-NEXT: vcvtph2ps %xmm1, %xmm1 -; F16C-NEXT: vpsrld $16, %xmm0, %xmm0 -; F16C-NEXT: vpextrw $0, %xmm0, %eax -; F16C-NEXT: movzwl %ax, %eax -; F16C-NEXT: vmovd %eax, %xmm0 +; F16C-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero ; F16C-NEXT: vcvtph2ps %xmm0, %xmm0 -; F16C-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; F16C-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; F16C-NEXT: vcvttps2dq %xmm0, %xmm0 ; F16C-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero ; F16C-NEXT: retq ; ; AVX512-LABEL: fptosi_2f16_to_4i32: ; AVX512: # %bb.0: -; AVX512-NEXT: vpextrw $0, %xmm0, %eax -; AVX512-NEXT: movzwl %ax, %eax -; AVX512-NEXT: vmovd %eax, %xmm1 +; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1 ; AVX512-NEXT: vcvtph2ps %xmm1, %xmm1 -; AVX512-NEXT: vpsrld $16, %xmm0, %xmm0 -; AVX512-NEXT: vpextrw $0, %xmm0, %eax -; AVX512-NEXT: movzwl %ax, %eax -; AVX512-NEXT: vmovd %eax, %xmm0 +; AVX512-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero ; AVX512-NEXT: vcvtph2ps %xmm0, %xmm0 -; AVX512-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; AVX512-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; AVX512-NEXT: vcvttps2dq %xmm0, %xmm0 ; AVX512-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero ; AVX512-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll b/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll index 71c4427..24113441 100644 --- a/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll +++ b/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll @@ -413,14 +413,8 @@ define half @test_v2f16(<2 x half> %a0) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; AVX512F-NEXT: vpsrld $16, %xmm0, %xmm1 -; AVX512F-NEXT: vpextrw $0, %xmm0, %eax -; AVX512F-NEXT: movzwl %ax, %eax -; AVX512F-NEXT: vmovd %eax, %xmm2 -; AVX512F-NEXT: vcvtph2ps %xmm2, %xmm2 -; AVX512F-NEXT: vpextrw $0, %xmm1, %eax -; AVX512F-NEXT: movzwl %ax, %eax -; AVX512F-NEXT: vmovd %eax, %xmm3 -; AVX512F-NEXT: vcvtph2ps %xmm3, %xmm3 +; AVX512F-NEXT: vcvtph2ps %xmm0, %xmm2 +; AVX512F-NEXT: vcvtph2ps %xmm1, %xmm3 ; AVX512F-NEXT: xorl %eax, %eax ; AVX512F-NEXT: vucomiss %xmm3, %xmm2 ; AVX512F-NEXT: movl $255, %ecx @@ -434,14 +428,8 @@ define half @test_v2f16(<2 x half> %a0) nounwind { ; AVX512VL-LABEL: test_v2f16: ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsrld $16, %xmm0, %xmm1 -; AVX512VL-NEXT: vpextrw $0, %xmm0, %eax -; AVX512VL-NEXT: movzwl %ax, %eax -; AVX512VL-NEXT: vmovd %eax, %xmm2 -; AVX512VL-NEXT: vcvtph2ps %xmm2, %xmm2 -; AVX512VL-NEXT: vpextrw $0, %xmm1, %eax -; AVX512VL-NEXT: movzwl %ax, %eax -; AVX512VL-NEXT: vmovd %eax, %xmm3 -; AVX512VL-NEXT: vcvtph2ps %xmm3, %xmm3 +; AVX512VL-NEXT: vcvtph2ps %xmm0, %xmm2 +; AVX512VL-NEXT: vcvtph2ps %xmm1, %xmm3 ; AVX512VL-NEXT: xorl %eax, %eax ; AVX512VL-NEXT: vucomiss %xmm3, %xmm2 ; AVX512VL-NEXT: movl $255, %ecx diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll b/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll index 0b2f9d6..edefb16 100644 --- a/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll +++ b/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll @@ -412,14 +412,8 @@ define half @test_v2f16(<2 x half> %a0) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; AVX512F-NEXT: vpsrld $16, %xmm0, %xmm1 -; AVX512F-NEXT: vpextrw $0, %xmm0, %eax -; AVX512F-NEXT: movzwl %ax, %eax -; AVX512F-NEXT: vmovd %eax, %xmm2 -; AVX512F-NEXT: vcvtph2ps %xmm2, %xmm2 -; AVX512F-NEXT: vpextrw $0, %xmm1, %eax -; AVX512F-NEXT: movzwl %ax, %eax -; AVX512F-NEXT: vmovd %eax, %xmm3 -; AVX512F-NEXT: vcvtph2ps %xmm3, %xmm3 +; AVX512F-NEXT: vcvtph2ps %xmm0, %xmm2 +; AVX512F-NEXT: vcvtph2ps %xmm1, %xmm3 ; AVX512F-NEXT: xorl %eax, %eax ; AVX512F-NEXT: vucomiss %xmm3, %xmm2 ; AVX512F-NEXT: movl $255, %ecx @@ -433,14 +427,8 @@ define half @test_v2f16(<2 x half> %a0) nounwind { ; AVX512VL-LABEL: test_v2f16: ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsrld $16, %xmm0, %xmm1 -; AVX512VL-NEXT: vpextrw $0, %xmm0, %eax -; AVX512VL-NEXT: movzwl %ax, %eax -; AVX512VL-NEXT: vmovd %eax, %xmm2 -; AVX512VL-NEXT: vcvtph2ps %xmm2, %xmm2 -; AVX512VL-NEXT: vpextrw $0, %xmm1, %eax -; AVX512VL-NEXT: movzwl %ax, %eax -; AVX512VL-NEXT: vmovd %eax, %xmm3 -; AVX512VL-NEXT: vcvtph2ps %xmm3, %xmm3 +; AVX512VL-NEXT: vcvtph2ps %xmm0, %xmm2 +; AVX512VL-NEXT: vcvtph2ps %xmm1, %xmm3 ; AVX512VL-NEXT: xorl %eax, %eax ; AVX512VL-NEXT: vucomiss %xmm3, %xmm2 ; AVX512VL-NEXT: movl $255, %ecx diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll index 468fec6..6360c68 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll @@ -2012,24 +2012,18 @@ define <4 x i32> @extract3_insert0_v4i32_7123(<4 x i32> %a0, <4 x i32> %a1) { ; SSE2-LABEL: extract3_insert0_v4i32_7123: ; SSE2: # %bb.0: ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3] -; SSE2-NEXT: movd %xmm1, %eax -; SSE2-NEXT: movd %eax, %xmm1 ; SSE2-NEXT: movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] ; SSE2-NEXT: retq ; ; SSE3-LABEL: extract3_insert0_v4i32_7123: ; SSE3: # %bb.0: ; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3] -; SSE3-NEXT: movd %xmm1, %eax -; SSE3-NEXT: movd %eax, %xmm1 ; SSE3-NEXT: movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] ; SSE3-NEXT: retq ; ; SSSE3-LABEL: extract3_insert0_v4i32_7123: ; SSSE3: # %bb.0: ; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3] -; SSSE3-NEXT: movd %xmm1, %eax -; SSSE3-NEXT: movd %eax, %xmm1 ; SSSE3-NEXT: movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] ; SSSE3-NEXT: retq ; diff --git a/llvm/test/MC/AArch64/cfi-bad-nesting-darwin.s b/llvm/test/MC/AArch64/cfi-bad-nesting-darwin.s index 235b7d4..3a5af86 100644 --- a/llvm/test/MC/AArch64/cfi-bad-nesting-darwin.s +++ b/llvm/test/MC/AArch64/cfi-bad-nesting-darwin.s @@ -8,6 +8,10 @@ .p2align 2 _locomotive: .cfi_startproc + ; An N_ALT_ENTRY symbol can be defined in the middle of a subsection, so + ; these are opted out of the .cfi_{start,end}proc nesting check. + .alt_entry _engineer +_engineer: ret ; It is invalid to have a non-private label between .cfi_startproc and @@ -17,7 +21,7 @@ _locomotive: .p2align 2 _caboose: ; DARWIN: [[#@LINE-1]]:1: error: non-private labels cannot appear between .cfi_startproc / .cfi_endproc pairs -; DARWIN: [[#@LINE-10]]:2: error: previous .cfi_startproc was here +; DARWIN: [[#@LINE-14]]:2: error: previous .cfi_startproc was here ret .cfi_endproc diff --git a/llvm/test/MC/AMDGPU/gfx11_unsupported.s b/llvm/test/MC/AMDGPU/gfx11_unsupported.s index bfca71a..f447263 100644 --- a/llvm/test/MC/AMDGPU/gfx11_unsupported.s +++ b/llvm/test/MC/AMDGPU/gfx11_unsupported.s @@ -2052,3 +2052,15 @@ global_atomic_cond_sub_u32 v0, v2, s[0:1] offset:64 global_atomic_ordered_add_b64 v0, v[2:3], s[0:1] offset:64 // CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU + +ds_subrev_u32 v1, v2 +// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU + +ds_subrev_rtn_u32 v5, v1, v2 +// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU + +ds_subrev_u64 v1, v[2:3] +// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU + +ds_subrev_rtn_u64 v[5:6], v1, v[2:3] +// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_ds_alias.s b/llvm/test/MC/AMDGPU/gfx12_asm_ds_alias.s index aa063c8..057e993 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_ds_alias.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_ds_alias.s @@ -27,5 +27,11 @@ ds_min_rtn_f64 v[5:6], v1, v[2:3] ds_subrev_u32 v1, v2 // GFX12: ds_rsub_u32 v1, v2 ; encoding: [0x00,0x00,0x08,0xd8,0x01,0x02,0x00,0x00] +ds_subrev_rtn_u32 v5, v1, v2 +// GFX12: ds_rsub_rtn_u32 v5, v1, v2 ; encoding: [0x00,0x00,0x88,0xd8,0x01,0x02,0x00,0x05] + ds_subrev_u64 v1, v[2:3] // GFX12: ds_rsub_u64 v1, v[2:3] ; encoding: [0x00,0x00,0x08,0xd9,0x01,0x02,0x00,0x00] + +ds_subrev_rtn_u64 v[5:6], v1, v[2:3] +// GFX12: ds_rsub_rtn_u64 v[5:6], v1, v[2:3] ; encoding: [0x00,0x00,0x88,0xd9,0x01,0x02,0x00,0x05] diff --git a/llvm/test/MC/ARM/thumbv8m.s b/llvm/test/MC/ARM/thumbv8m.s index f03dd03..0e9ab4a 100644 --- a/llvm/test/MC/ARM/thumbv8m.s +++ b/llvm/test/MC/ARM/thumbv8m.s @@ -184,13 +184,13 @@ ttat r0, r1 // 'Lazy Load/Store Multiple' // UNDEF-BASELINE: error: instruction requires: armv8m.main -// CHECK-MAINLINE: vlldm r5, {d0 - d15} @ encoding: [0x35,0xec,0x00,0x0a] -// CHECK-MAINLINE_DSP: vlldm r5, {d0 - d15} @ encoding: [0x35,0xec,0x00,0x0a] +// CHECK-MAINLINE: vlldm r5 @ encoding: [0x35,0xec,0x00,0x0a] +// CHECK-MAINLINE_DSP: vlldm r5 @ encoding: [0x35,0xec,0x00,0x0a] vlldm r5 // UNDEF-BASELINE: error: instruction requires: armv8m.main -// CHECK-MAINLINE: vlstm r10, {d0 - d15} @ encoding: [0x2a,0xec,0x00,0x0a] -// CHECK-MAINLINE_DSP: vlstm r10, {d0 - d15} @ encoding: [0x2a,0xec,0x00,0x0a] +// CHECK-MAINLINE: vlstm r10 @ encoding: [0x2a,0xec,0x00,0x0a] +// CHECK-MAINLINE_DSP: vlstm r10 @ encoding: [0x2a,0xec,0x00,0x0a] vlstm r10 // New SYSm's diff --git a/llvm/test/MC/ARM/vlstm-vlldm-8.1m.s b/llvm/test/MC/ARM/vlstm-vlldm-8.1m.s deleted file mode 100644 index 4e35883..0000000 --- a/llvm/test/MC/ARM/vlstm-vlldm-8.1m.s +++ /dev/null @@ -1,11 +0,0 @@ -// RUN: llvm-mc -triple=armv8.1m.main-arm-none-eabi -mcpu=generic -show-encoding %s \ -// RUN: | FileCheck --check-prefixes=CHECK %s - -// RUN: llvm-mc -triple=thumbv8.1m.main-none-eabi -mcpu=generic -show-encoding %s \ -// RUN: | FileCheck --check-prefixes=CHECK %s - -vlstm r8, {d0 - d31} -// CHECK: vlstm r8, {d0 - d31} @ encoding: [0x28,0xec,0x80,0x0a] - -vlldm r8, {d0 - d31} -// CHECK: vlldm r8, {d0 - d31} @ encoding: [0x38,0xec,0x80,0x0a] diff --git a/llvm/test/MC/ARM/vlstm-vlldm-8m.s b/llvm/test/MC/ARM/vlstm-vlldm-8m.s deleted file mode 100644 index bbc9531..0000000 --- a/llvm/test/MC/ARM/vlstm-vlldm-8m.s +++ /dev/null @@ -1,17 +0,0 @@ -// RUN: llvm-mc -triple=armv8m.main-arm-none-eabi -mcpu=generic -show-encoding %s \ -// RUN: | FileCheck --check-prefixes=CHECK %s - -// RUN: llvm-mc -triple=thumbv8m.main-none-eabi -mcpu=generic -show-encoding %s \ -// RUN: | FileCheck --check-prefixes=CHECK %s - -vlstm r8, {d0 - d15} -// CHECK: vlstm r8, {d0 - d15} @ encoding: [0x28,0xec,0x00,0x0a] - -vlldm r8, {d0 - d15} -// CHECK: vlldm r8, {d0 - d15} @ encoding: [0x38,0xec,0x00,0x0a] - -vlstm r8 -// CHECK: vlstm r8, {d0 - d15} @ encoding: [0x28,0xec,0x00,0x0a] - -vlldm r8 -// CHECK: vlldm r8, {d0 - d15} @ encoding: [0x38,0xec,0x00,0x0a] diff --git a/llvm/test/MC/ARM/vlstm-vlldm-diag.s b/llvm/test/MC/ARM/vlstm-vlldm-diag.s deleted file mode 100644 index b57f535..0000000 --- a/llvm/test/MC/ARM/vlstm-vlldm-diag.s +++ /dev/null @@ -1,61 +0,0 @@ -// RUN: not llvm-mc -triple=armv8.1m.main-arm-none-eabi -mcpu=generic -show-encoding %s 2>&1 >/dev/null \ -// RUN: | FileCheck --check-prefixes=ERR %s - -// RUN: not llvm-mc -triple=armv8.1m.main-arm-none-eabi -mcpu=generic -show-encoding %s 2>&1 >/dev/null \ -// RUN: | FileCheck --check-prefixes=ERRT2 %s - -vlstm r8, {d0 - d11} -// ERR: error: operand must be exactly {d0-d15} (T1) or {d0-d31} (T2) -// ERR-NEXT: vlstm r8, {d0 - d11} - -vlldm r8, {d0 - d11} -// ERR: error: operand must be exactly {d0-d15} (T1) or {d0-d31} (T2) -// ERR-NEXT: vlldm r8, {d0 - d11} - -vlstm r8, {d3 - d15} -// ERR: error: operand must be exactly {d0-d15} (T1) or {d0-d31} (T2) -// ERR-NEXT: vlstm r8, {d3 - d15} - -vlldm r8, {d3 - d15} -// ERR: error: operand must be exactly {d0-d15} (T1) or {d0-d31} (T2) -// ERR-NEXT: vlldm r8, {d3 - d15} - -vlstm r8, {d0 - d29} -// ERR: error: operand must be exactly {d0-d15} (T1) or {d0-d31} (T2) -// ERR-NEXT: vlstm r8, {d0 - d29} - -vlldm r8, {d0 - d29} -// ERR: error: operand must be exactly {d0-d15} (T1) or {d0-d31} (T2) -// ERR-NEXT: vlldm r8, {d0 - d29} - -vlstm r8, {d3 - d31} -// ERR: error: operand must be exactly {d0-d15} (T1) or {d0-d31} (T2) -// ERR-NEXT: vlstm r8, {d3 - d31} - -vlldm r8, {d3 - d31} -// ERR: error: operand must be exactly {d0-d15} (T1) or {d0-d31} (T2) -// ERR-NEXT: vlldm r8, {d3 - d31} - -vlstm r8, {d0 - d35} -// ERR: error: register expected -// ERR-NEXT: vlstm r8, {d0 - d35} - -vlldm r8, {d0 - d35} -// ERR: error: register expected -// ERR-NEXT: vlldm r8, {d0 - d35} - -vlstm pc -// ERR: error: operand must be a register in range [r0, r14] -// ERR-NEXT: vlstm pc - -vlldm pc -// ERR: error: operand must be a register in range [r0, r14] -// ERR-NEXT: vlldm pc - -vlstm pc -// ERRT2: error: operand must be a register in range [r0, r14] -// ERRT2-NEXT: vlstm pc - -vlldm pc -// ERRT2: error: operand must be a register in range [r0, r14] -// ERRT2-NEXT: vlldm pc
\ No newline at end of file diff --git a/llvm/test/MC/Disassembler/ARM/armv8.1m-vlldm_vlstm-8.1.main.txt b/llvm/test/MC/Disassembler/ARM/armv8.1m-vlldm_vlstm-8.1.main.txt deleted file mode 100644 index 6b988245..0000000 --- a/llvm/test/MC/Disassembler/ARM/armv8.1m-vlldm_vlstm-8.1.main.txt +++ /dev/null @@ -1,11 +0,0 @@ -// RUN: llvm-mc -triple=armv8.1m.main-arm-none-eabi -mcpu=generic -show-encoding -disassemble %s \ -// RUN: | FileCheck %s --check-prefixes=CHECK-DISS - -// RUN: llvm-mc -triple=thumbv8.1m.main-none-eabi -mcpu=generic -show-encoding -disassemble %s \ -// RUN: | FileCheck %s --check-prefixes=CHECK-DISS - -[0x28,0xec,0x80,0x0a] -// CHECK-DISS: vlstm r8, {d0 - d31} @ encoding: [0x28,0xec,0x80,0x0a] - -[0x38,0xec,0x80,0x0a] -// CHECK-DISS: vlldm r8, {d0 - d31} @ encoding: [0x38,0xec,0x80,0x0a]
\ No newline at end of file diff --git a/llvm/test/MC/Disassembler/ARM/armv8.1m-vlldm_vlstm-8.main.txt b/llvm/test/MC/Disassembler/ARM/armv8.1m-vlldm_vlstm-8.main.txt deleted file mode 100644 index 1e28d52..0000000 --- a/llvm/test/MC/Disassembler/ARM/armv8.1m-vlldm_vlstm-8.main.txt +++ /dev/null @@ -1,17 +0,0 @@ -// RUN: llvm-mc -triple=armv8m.main-arm-none-eabi -mcpu=generic -show-encoding -disassemble %s \ -// RUN: | FileCheck %s --check-prefixes=CHECK-DISS - -// RUN: llvm-mc -triple=thumbv8m.main-none-eabi -mcpu=generic -show-encoding -disassemble %s \ -// RUN: | FileCheck %s --check-prefixes=CHECK-DISS - -[0x28,0xec,0x00,0x0a] -// CHECK-DISS: vlstm r8, {d0 - d15} @ encoding: [0x28,0xec,0x00,0x0a] - -[0x38,0xec,0x00,0x0a] -// CHECK-DISS: vlldm r8, {d0 - d15} @ encoding: [0x38,0xec,0x00,0x0a] - -[0x28,0xec,0x00,0x0a] -// CHECK-DISS: vlstm r8, {d0 - d15} @ encoding: [0x28,0xec,0x00,0x0a] - -[0x38,0xec,0x00,0x0a] -// CHECK-DISS: vlldm r8, {d0 - d15} @ encoding: [0x38,0xec,0x00,0x0a]
\ No newline at end of file diff --git a/llvm/test/MC/Disassembler/X86/apx/IgnoreW.txt b/llvm/test/MC/Disassembler/X86/apx/IgnoreW.txt new file mode 100644 index 0000000..df41bdf --- /dev/null +++ b/llvm/test/MC/Disassembler/X86/apx/IgnoreW.txt @@ -0,0 +1,118 @@ +# RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s --check-prefixes=ATT +# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL + +## invpcid + +# ATT: invpcid 123(%rax,%rbx,4), %r9 +# INTEL: invpcid r9, xmmword ptr [rax + 4*rbx + 123] +0x62,0x74,0xfe,0x08,0xf2,0x4c,0x98,0x7b + +# ATT: invpcid 291(%r28,%r29,4), %r19 +# INTEL: invpcid r19, xmmword ptr [r28 + 4*r29 + 291] +0x62,0x8c,0xfa,0x08,0xf2,0x9c,0xac,0x23,0x01,0x00,0x00 + +## invept + +# ATT: invept 291(%r28,%r29,4), %r19 +# INTEL: invept r19, xmmword ptr [r28 + 4*r29 + 291] +0x62,0x8c,0xfa,0x08,0xf0,0x9c,0xac,0x23,0x01,0x00,0x00 + +# ATT: invept 123(%rax,%rbx,4), %r9 +# INTEL: invept r9, xmmword ptr [rax + 4*rbx + 123] +0x62,0x74,0xfe,0x08,0xf0,0x4c,0x98,0x7b + +## invvpid + +# ATT: invvpid 291(%r28,%r29,4), %r19 +# INTEL: invvpid r19, xmmword ptr [r28 + 4*r29 + 291] +0x62,0x8c,0xfa,0x08,0xf1,0x9c,0xac,0x23,0x01,0x00,0x00 + +# ATT: invvpid 123(%rax,%rbx,4), %r9 +# INTEL: invvpid r9, xmmword ptr [rax + 4*rbx + 123] +0x62,0x74,0xfe,0x08,0xf1,0x4c,0x98,0x7b + +## adc + +# ATT: {evex} adcb $123, %bl +# INTEL: {evex} adc bl, 123 +0x62,0xf4,0xfc,0x08,0x80,0xd3,0x7b + +# ATT: adcb $123, %bl, %cl +# INTEL: adc cl, bl, 123 +0x62,0xf4,0xf4,0x18,0x80,0xd3,0x7b + +# ATT: adcb $123, %r16b +# INTEL: adc r16b, 123 +0xd5,0x18,0x80,0xd0,0x7b + +## add + +# ATT: {evex} addb $123, %bl +# INTEL: {evex} add bl, 123 +0x62,0xf4,0xfc,0x08,0x80,0xc3,0x7b + +# ATT: {nf} addb $123, %bl +# INTEL: {nf} add bl, 123 +0x62,0xf4,0xfc,0x0c,0x80,0xc3,0x7b + +# ATT: addb $123, %bl, %cl +# INTEL: add cl, bl, 123 +0x62,0xf4,0xf4,0x18,0x80,0xc3,0x7b + +# ATT: {nf} addb $123, %bl, %cl +# INTEL: {nf} add cl, bl, 123 +0x62,0xf4,0xf4,0x1c,0x80,0xc3,0x7b + +# ATT: addb $123, %r16b +# INTEL: add r16b, 123 +0xd5,0x18,0x80,0xc0,0x7b + +## inc + +# ATT: {evex} incb %bl +# INTEL: {evex} inc bl +0x62,0xf4,0xfc,0x08,0xfe,0xc3 + +# ATT: {nf} incb %bl +# INTEL: {nf} inc bl +0x62,0xf4,0xfc,0x0c,0xfe,0xc3 + +# ATT: incb %bl, %bl +# INTEL: inc bl, bl +0x62,0xf4,0xe4,0x18,0xfe,0xc3 + +# ATT: {nf} incb %bl, %bl +# INTEL: {nf} inc bl, bl +0x62,0xf4,0xe4,0x1c,0xfe,0xc3 + +# ATT: incb %r16b +# INTEL: inc r16b +0xd5,0x18,0xfe,0xc0 + +## mul + +# ATT: {evex} mulb %bl +# INTEL: {evex} mul bl +0x62,0xf4,0xfc,0x08,0xf6,0xe3 + +# ATT: {nf} mulb %bl +# INTEL: {nf} mul bl +0x62,0xf4,0xfc,0x0c,0xf6,0xe3 + +# ATT: mulb %r16b +# INTEL: mul r16b +0xd5,0x18,0xf6,0xe0 + +## imul + +# ATT: {evex} imulb %bl +# INTEL: {evex} imul bl +0x62,0xf4,0xfc,0x08,0xf6,0xeb + +# ATT: {nf} imulb %bl +# INTEL: {nf} imul bl +0x62,0xf4,0xfc,0x0c,0xf6,0xeb + +# ATT: imulb %r16b +# INTEL: imul r16b +0xd5,0x18,0xf6,0xe8 diff --git a/llvm/test/MC/RISCV/rv32zacas-invalid.s b/llvm/test/MC/RISCV/rv32zacas-invalid.s index b86246c..11d20da 100644 --- a/llvm/test/MC/RISCV/rv32zacas-invalid.s +++ b/llvm/test/MC/RISCV/rv32zacas-invalid.s @@ -1,4 +1,4 @@ -# RUN: not llvm-mc -triple riscv32 -mattr=+experimental-zacas < %s 2>&1 | FileCheck %s +# RUN: not llvm-mc -triple riscv32 -mattr=+zacas < %s 2>&1 | FileCheck %s # Non-zero offsets not supported for the third operand (rs1). amocas.w a1, a3, 1(a5) # CHECK: :[[@LINE]]:18: error: optional integer offset must be 0 diff --git a/llvm/test/MC/RISCV/rv32zacas-valid.s b/llvm/test/MC/RISCV/rv32zacas-valid.s index d80b963..05a9cdd 100644 --- a/llvm/test/MC/RISCV/rv32zacas-valid.s +++ b/llvm/test/MC/RISCV/rv32zacas-valid.s @@ -1,12 +1,12 @@ -# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-zacas -riscv-no-aliases -show-encoding \ +# RUN: llvm-mc %s -triple=riscv32 -mattr=+zacas -riscv-no-aliases -show-encoding \ # RUN: | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s -# RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-zacas -riscv-no-aliases -show-encoding \ +# RUN: llvm-mc %s -triple=riscv64 -mattr=+zacas -riscv-no-aliases -show-encoding \ # RUN: | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s -# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+experimental-zacas < %s \ -# RUN: | llvm-objdump --mattr=+experimental-zacas -M no-aliases -d -r - \ +# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+zacas < %s \ +# RUN: | llvm-objdump --mattr=+zacas -M no-aliases -d -r - \ # RUN: | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s -# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+experimental-zacas < %s \ -# RUN: | llvm-objdump --mattr=+experimental-zacas -M no-aliases -d -r - \ +# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+zacas < %s \ +# RUN: | llvm-objdump --mattr=+zacas -M no-aliases -d -r - \ # RUN: | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s # RUN: not llvm-mc -triple=riscv32 -mattr=+a -show-encoding %s 2>&1 \ # RUN: | FileCheck %s --check-prefix=CHECK-ERROR diff --git a/llvm/test/MC/RISCV/rv64zacas-valid.s b/llvm/test/MC/RISCV/rv64zacas-valid.s index 843401b..694f43b 100644 --- a/llvm/test/MC/RISCV/rv64zacas-valid.s +++ b/llvm/test/MC/RISCV/rv64zacas-valid.s @@ -1,7 +1,7 @@ -# RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-zacas -riscv-no-aliases -show-encoding \ +# RUN: llvm-mc %s -triple=riscv64 -mattr=+zacas -riscv-no-aliases -show-encoding \ # RUN: | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s -# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+experimental-zacas < %s \ -# RUN: | llvm-objdump --mattr=+experimental-zacas -M no-aliases -d -r - \ +# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+zacas < %s \ +# RUN: | llvm-objdump --mattr=+zacas -M no-aliases -d -r - \ # RUN: | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s # RUN: not llvm-mc -triple=riscv64 -mattr=+a -show-encoding %s 2>&1 \ # RUN: | FileCheck %s --check-prefix=CHECK-ERROR diff --git a/llvm/test/MC/RISCV/rvzabha-zacas-valid.s b/llvm/test/MC/RISCV/rvzabha-zacas-valid.s index 8ad2f99..f1f705e 100644 --- a/llvm/test/MC/RISCV/rvzabha-zacas-valid.s +++ b/llvm/test/MC/RISCV/rvzabha-zacas-valid.s @@ -1,12 +1,12 @@ -# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-zabha,+experimental-zacas -riscv-no-aliases -show-encoding \ +# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-zabha,+zacas -riscv-no-aliases -show-encoding \ # RUN: | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s -# RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-zabha,+experimental-zacas -riscv-no-aliases -show-encoding \ +# RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-zabha,+zacas -riscv-no-aliases -show-encoding \ # RUN: | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s -# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+experimental-zabha,+experimental-zacas < %s \ -# RUN: | llvm-objdump --mattr=+experimental-zabha,+experimental-zacas -M no-aliases -d -r - \ +# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+experimental-zabha,+zacas < %s \ +# RUN: | llvm-objdump --mattr=+experimental-zabha,+zacas -M no-aliases -d -r - \ # RUN: | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s -# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+experimental-zabha,+experimental-zacas < %s \ -# RUN: | llvm-objdump --mattr=+experimental-zabha,+experimental-zacas -M no-aliases -d -r - \ +# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+experimental-zabha,+zacas < %s \ +# RUN: | llvm-objdump --mattr=+experimental-zabha,+zacas -M no-aliases -d -r - \ # RUN: | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s # RUN: not llvm-mc -triple=riscv32 -mattr=+experimental-zabha -show-encoding %s 2>&1 \ # RUN: | FileCheck %s --check-prefix=CHECK-ERROR diff --git a/llvm/test/ThinLTO/X86/visibility-elf.ll b/llvm/test/ThinLTO/X86/visibility-elf.ll index aa11c3e..fc7439b 100644 --- a/llvm/test/ThinLTO/X86/visibility-elf.ll +++ b/llvm/test/ThinLTO/X86/visibility-elf.ll @@ -36,12 +36,12 @@ declare void @ext(ptr) ;; Currently the visibility is not propagated onto an unimported function, ;; because we don't have summaries for declarations. ; CHECK: declare extern_weak void @not_imported() -; CHECK: define available_externally hidden void @hidden_def_ref() !thinlto_src_module !0 -; CHECK: define available_externally hidden void @hidden_def_weak_ref() !thinlto_src_module !0 +; CHECK: define available_externally hidden void @hidden_def_ref() !thinlto_src_module !0 !thinlto_src_file !1 +; CHECK: define available_externally hidden void @hidden_def_weak_ref() !thinlto_src_module !0 !thinlto_src_file !1 ;; This can be hidden, but we cannot communicate the declaration's visibility ;; to other modules because declarations don't have summaries, and the IRLinker ;; overrides it when importing the protected def. -; CHECK: define available_externally protected void @protected_def_hidden_ref() !thinlto_src_module !0 +; CHECK: define available_externally protected void @protected_def_hidden_ref() !thinlto_src_module !0 !thinlto_src_file !1 ; CHECK2: define hidden i32 @hidden_def_weak_def() ; CHECK2: define protected void @protected_def_weak_def() diff --git a/llvm/test/ThinLTO/X86/visibility-macho.ll b/llvm/test/ThinLTO/X86/visibility-macho.ll index d41ab4f..1a48b47 100644 --- a/llvm/test/ThinLTO/X86/visibility-macho.ll +++ b/llvm/test/ThinLTO/X86/visibility-macho.ll @@ -30,8 +30,8 @@ declare void @ext(ptr) ;; Currently the visibility is not propagated onto an unimported function, ;; because we don't have summaries for declarations. ; CHECK: declare extern_weak dso_local void @not_imported() -; CHECK: define available_externally hidden void @hidden_def_ref() !thinlto_src_module !0 -; CHECK: define available_externally hidden void @hidden_def_weak_ref() !thinlto_src_module !0 +; CHECK: define available_externally hidden void @hidden_def_ref() !thinlto_src_module !0 !thinlto_src_file !1 +; CHECK: define available_externally hidden void @hidden_def_weak_ref() !thinlto_src_module !0 !thinlto_src_file !1 ; CHECK2: define hidden i32 @hidden_def_weak_def() ; CHECK2: define hidden void @hidden_def_ref() diff --git a/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll b/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll index 47b2dda..dd9310f 100644 --- a/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll +++ b/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll @@ -9,6 +9,11 @@ ; CHECK-SAME: !DIExpression(DW_OP_plus_uconst, [[OffsetX:[0-9]*]])) ; ^ No deref at the end, as this variable ("x") is an array; ; its value is its address. The entire array is in the frame. +; CHECK: call void @llvm.dbg.assign(metadata ptr %[[frame]] +; CHECK-SAME: !DIExpression(DW_OP_plus_uconst, [[OffsetX]]) +;; FIXME: Should we be updating the addresses on assigns here as well? +; CHECK-SAME: , metadata ptr %[[frame]], metadata !DIExpression()) + ; CHECK: call void @llvm.dbg.value(metadata ptr %[[frame]] ; CHECK-SAME: !DIExpression(DW_OP_plus_uconst, [[OffsetSpill:[0-9]*]], DW_OP_deref)) ; CHECK: call void @llvm.dbg.value(metadata ptr %[[frame]] @@ -78,6 +83,7 @@ init.ready: ; preds = %init.suspend, %coro %i.init.ready.inc = add nsw i32 0, 1 call void @llvm.dbg.value(metadata i32 %i.init.ready.inc, metadata !6, metadata !DIExpression()), !dbg !11 call void @llvm.dbg.value(metadata ptr %x, metadata !12, metadata !DIExpression()), !dbg !17 + call void @llvm.dbg.assign(metadata ptr %x, metadata !12, metadata !DIExpression(), metadata !30, metadata ptr %x, metadata !DIExpression()), !dbg !17 call void @llvm.memset.p0.i64(ptr align 16 %x, i8 0, i64 40, i1 false), !dbg !17 call void @print(i32 %i.init.ready.inc) %ready.again = call zeroext i1 @await_ready() @@ -250,3 +256,4 @@ attributes #4 = { argmemonly nofree nosync nounwind willreturn writeonly } !21 = !DILocation(line: 43, column: 3, scope: !7) !22 = !DILocation(line: 43, column: 8, scope: !7) !23 = !DILocalVariable(name: "produced", scope: !7, file: !1, line:24, type: !10) +!30 = distinct !DIAssignID()
\ No newline at end of file diff --git a/llvm/test/Transforms/FunctionImport/funcimport.ll b/llvm/test/Transforms/FunctionImport/funcimport.ll index 0129825..a0968a6 100644 --- a/llvm/test/Transforms/FunctionImport/funcimport.ll +++ b/llvm/test/Transforms/FunctionImport/funcimport.ll @@ -57,7 +57,7 @@ declare void @linkoncealias(...) #1 ; CHECK-DAG: define available_externally void @linkoncealias() ; INSTLIMDEF-DAG: Import referencestatics -; INSTLIMDEF-DAG: define available_externally i32 @referencestatics(i32 %i) !thinlto_src_module !0 { +; INSTLIMDEF-DAG: define available_externally i32 @referencestatics(i32 %i) !thinlto_src_module !0 !thinlto_src_file !1 { ; INSTLIM5-DAG: declare i32 @referencestatics(...) declare i32 @referencestatics(...) #1 @@ -66,27 +66,27 @@ declare i32 @referencestatics(...) #1 ; Ensure that the call is to the properly-renamed function. ; INSTLIMDEF-DAG: Import staticfunc ; INSTLIMDEF-DAG: %call = call i32 @staticfunc.llvm. -; INSTLIMDEF-DAG: define available_externally hidden i32 @staticfunc.llvm.{{.*}} !thinlto_src_module !0 { +; INSTLIMDEF-DAG: define available_externally hidden i32 @staticfunc.llvm.{{.*}} !thinlto_src_module !0 !thinlto_src_file !1 { ; INSTLIMDEF-DAG: Import referenceglobals -; CHECK-DAG: define available_externally i32 @referenceglobals(i32 %i) !thinlto_src_module !0 { +; CHECK-DAG: define available_externally i32 @referenceglobals(i32 %i) !thinlto_src_module !0 !thinlto_src_file !1 { declare i32 @referenceglobals(...) #1 ; The import of referenceglobals will expose call to globalfunc1 that ; should in turn be imported. ; INSTLIMDEF-DAG: Import globalfunc1 -; CHECK-DAG: define available_externally void @globalfunc1() !thinlto_src_module !0 +; CHECK-DAG: define available_externally void @globalfunc1() !thinlto_src_module !0 !thinlto_src_file !1 ; INSTLIMDEF-DAG: Import referencecommon -; CHECK-DAG: define available_externally i32 @referencecommon(i32 %i) !thinlto_src_module !0 { +; CHECK-DAG: define available_externally i32 @referencecommon(i32 %i) !thinlto_src_module !0 !thinlto_src_file !1 { declare i32 @referencecommon(...) #1 ; INSTLIMDEF-DAG: Import setfuncptr -; CHECK-DAG: define available_externally void @setfuncptr() !thinlto_src_module !0 { +; CHECK-DAG: define available_externally void @setfuncptr() !thinlto_src_module !0 !thinlto_src_file !1 { declare void @setfuncptr(...) #1 ; INSTLIMDEF-DAG: Import callfuncptr -; CHECK-DAG: define available_externally void @callfuncptr() !thinlto_src_module !0 { +; CHECK-DAG: define available_externally void @callfuncptr() !thinlto_src_module !0 !thinlto_src_file !1 { declare void @callfuncptr(...) #1 ; Ensure that all uses of local variable @P which has used in setfuncptr @@ -97,7 +97,7 @@ declare void @callfuncptr(...) #1 ; Ensure that @referencelargelinkonce definition is pulled in, but later we ; also check that the linkonceodr function is not. -; CHECK-DAG: define available_externally void @referencelargelinkonce() !thinlto_src_module !0 { +; CHECK-DAG: define available_externally void @referencelargelinkonce() !thinlto_src_module !0 !thinlto_src_file !1 { ; INSTLIM5-DAG: declare void @linkonceodr() declare void @referencelargelinkonce(...) @@ -110,13 +110,13 @@ declare void @weakfunc(...) #1 declare void @linkoncefunc2(...) #1 ; INSTLIMDEF-DAG: Import funcwithpersonality -; INSTLIMDEF-DAG: define available_externally hidden void @funcwithpersonality.llvm.{{.*}}() personality ptr @__gxx_personality_v0 !thinlto_src_module !0 { +; INSTLIMDEF-DAG: define available_externally hidden void @funcwithpersonality.llvm.{{.*}}() personality ptr @__gxx_personality_v0 !thinlto_src_module !0 !thinlto_src_file !1 { ; INSTLIM5-DAG: declare hidden void @funcwithpersonality.llvm.{{.*}}() ; We can import variadic functions without a va_start, since the inliner ; can handle them. ; INSTLIMDEF-DAG: Import variadic_no_va_start -; CHECK-DAG: define available_externally void @variadic_no_va_start(...) !thinlto_src_module !0 { +; CHECK-DAG: define available_externally void @variadic_no_va_start(...) !thinlto_src_module !0 !thinlto_src_file !1 { declare void @variadic_no_va_start(...) ; We can import variadic functions with a va_start, since the inliner @@ -128,7 +128,8 @@ declare void @variadic_va_start(...) ; INSTLIMDEF-DAG: 15 function-import - Number of functions imported ; INSTLIMDEF-DAG: 4 function-import - Number of global variables imported -; CHECK-DAG: !0 = !{!"{{.*}}/Inputs/funcimport.ll"} +; CHECK-DAG: !0 = !{!"{{.*}}.bc"} +; CHECK-DAG: !1 = !{!"{{.*}}/Inputs/funcimport.ll"} ; The actual GUID values will depend on path to test. ; GUID-DAG: GUID {{.*}} is weakalias diff --git a/llvm/test/Transforms/Inline/inline_stats.ll b/llvm/test/Transforms/Inline/inline_stats.ll index c779054..41c12b3 100644 --- a/llvm/test/Transforms/Inline/inline_stats.ll +++ b/llvm/test/Transforms/Inline/inline_stats.ll @@ -44,7 +44,7 @@ define void @internal3() { declare void @external_decl() -define void @external1() alwaysinline !thinlto_src_module !0 { +define void @external1() alwaysinline !thinlto_src_module !0 !thinlto_src_file !1 { call fastcc void @internal2() call fastcc void @external2(); call void @external_decl(); @@ -87,7 +87,7 @@ define void @external_big() noinline !thinlto_src_module !1 { } ; It should not be imported, but it should not break anything. -define void @external_notcalled() !thinlto_src_module !0 { +define void @external_notcalled() !thinlto_src_module !0 !thinlto_src_file !1 { call void @external_notcalled() ret void } diff --git a/llvm/test/Transforms/InstCombine/maxnum.ll b/llvm/test/Transforms/InstCombine/maxnum.ll index 87288b1..e140a5b 100644 --- a/llvm/test/Transforms/InstCombine/maxnum.ll +++ b/llvm/test/Transforms/InstCombine/maxnum.ll @@ -66,7 +66,7 @@ define float @constant_fold_maxnum_f32_p0_n0() { define float @constant_fold_maxnum_f32_n0_p0() { ; CHECK-LABEL: @constant_fold_maxnum_f32_n0_p0( -; CHECK-NEXT: ret float -0.000000e+00 +; CHECK-NEXT: ret float 0.000000e+00 ; %x = call float @llvm.maxnum.f32(float -0.0, float 0.0) ret float %x diff --git a/llvm/test/Transforms/InstCombine/minnum.ll b/llvm/test/Transforms/InstCombine/minnum.ll index 8050f07..cc6171b 100644 --- a/llvm/test/Transforms/InstCombine/minnum.ll +++ b/llvm/test/Transforms/InstCombine/minnum.ll @@ -60,7 +60,7 @@ define float @constant_fold_minnum_f32_p0_p0() { define float @constant_fold_minnum_f32_p0_n0() { ; CHECK-LABEL: @constant_fold_minnum_f32_p0_n0( -; CHECK-NEXT: ret float 0.000000e+00 +; CHECK-NEXT: ret float -0.000000e+00 ; %x = call float @llvm.minnum.f32(float 0.0, float -0.0) ret float %x @@ -199,7 +199,7 @@ define float @minnum_f32_1_minnum_p0_val_fmf3(float %x) { define float @minnum_f32_p0_minnum_val_n0(float %x) { ; CHECK-LABEL: @minnum_f32_p0_minnum_val_n0( -; CHECK-NEXT: [[Z:%.*]] = call float @llvm.minnum.f32(float [[X:%.*]], float 0.000000e+00) +; CHECK-NEXT: [[Z:%.*]] = call float @llvm.minnum.f32(float [[X:%.*]], float -0.000000e+00) ; CHECK-NEXT: ret float [[Z]] ; %y = call float @llvm.minnum.f32(float %x, float -0.0) diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/min-max.ll b/llvm/test/Transforms/InstSimplify/ConstProp/min-max.ll index a5f5d4e..9120649 100644 --- a/llvm/test/Transforms/InstSimplify/ConstProp/min-max.ll +++ b/llvm/test/Transforms/InstSimplify/ConstProp/min-max.ll @@ -49,6 +49,38 @@ define float @minnum_float() { ret float %1 } +define float @minnum_float_p0_n0() { +; CHECK-LABEL: @minnum_float_p0_n0( +; CHECK-NEXT: ret float -0.000000e+00 +; + %min = call float @llvm.minnum.f32(float 0.0, float -0.0) + ret float %min +} + +define float @minnum_float_n0_p0() { +; CHECK-LABEL: @minnum_float_n0_p0( +; CHECK-NEXT: ret float -0.000000e+00 +; + %min = call float @llvm.minnum.f32(float -0.0, float 0.0) + ret float %min +} + +define float @minnum_float_p0_qnan() { +; CHECK-LABEL: @minnum_float_p0_qnan( +; CHECK-NEXT: ret float 0.000000e+00 +; + %min = call float @llvm.minnum.f32(float 0.0, float 0x7FF8000000000000) + ret float %min +} + +define float @minnum_float_qnan_p0() { +; CHECK-LABEL: @minnum_float_qnan_p0( +; CHECK-NEXT: ret float 0.000000e+00 +; + %min = call float @llvm.minnum.f32(float 0x7FF8000000000000, float 0.0) + ret float %min +} + define bfloat @minnum_bfloat() { ; CHECK-LABEL: @minnum_bfloat( ; CHECK-NEXT: ret bfloat 0xR40A0 @@ -95,7 +127,7 @@ define <4 x half> @minnum_half_vec() { define <4 x float> @minnum_float_zeros_vec() { ; CHECK-LABEL: @minnum_float_zeros_vec( -; CHECK-NEXT: ret <4 x float> <float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00> +; CHECK-NEXT: ret <4 x float> <float 0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00> ; %1 = call <4 x float> @llvm.minnum.v4f32(<4 x float> <float 0.0, float -0.0, float 0.0, float -0.0>, <4 x float> <float 0.0, float 0.0, float -0.0, float -0.0>) ret <4 x float> %1 @@ -109,6 +141,38 @@ define float @maxnum_float() { ret float %1 } +define float @maxnum_float_p0_n0() { +; CHECK-LABEL: @maxnum_float_p0_n0( +; CHECK-NEXT: ret float 0.000000e+00 +; + %max = call float @llvm.maxnum.f32(float 0.0, float -0.0) + ret float %max +} + +define float @maxnum_float_n0_p0() { +; CHECK-LABEL: @maxnum_float_n0_p0( +; CHECK-NEXT: ret float 0.000000e+00 +; + %max = call float @llvm.maxnum.f32(float -0.0, float 0.0) + ret float %max +} + +define float @maxnum_float_p0_qnan() { +; CHECK-LABEL: @maxnum_float_p0_qnan( +; CHECK-NEXT: ret float 0.000000e+00 +; + %max = call float @llvm.maxnum.f32(float 0.0, float 0x7FF8000000000000) + ret float %max +} + +define float @maxnum_float_qnan_p0() { +; CHECK-LABEL: @maxnum_float_qnan_p0( +; CHECK-NEXT: ret float 0.000000e+00 +; + %max = call float @llvm.maxnum.f32(float 0x7FF8000000000000, float 0.0) + ret float %max +} + define bfloat @maxnum_bfloat() { ; CHECK-LABEL: @maxnum_bfloat( ; CHECK-NEXT: ret bfloat 0xR4228 @@ -155,7 +219,7 @@ define <4 x half> @maxnum_half_vec() { define <4 x float> @maxnum_float_zeros_vec() { ; CHECK-LABEL: @maxnum_float_zeros_vec( -; CHECK-NEXT: ret <4 x float> <float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00> +; CHECK-NEXT: ret <4 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float -0.000000e+00> ; %1 = call <4 x float> @llvm.maxnum.v4f32(<4 x float> <float 0.0, float -0.0, float 0.0, float -0.0>, <4 x float> <float 0.0, float 0.0, float -0.0, float -0.0>) ret <4 x float> %1 diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/interleave_IC.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/interleave_IC.ll index 7121c85..c12b3b1 100644 --- a/llvm/test/Transforms/LoopVectorize/PowerPC/interleave_IC.ll +++ b/llvm/test/Transforms/LoopVectorize/PowerPC/interleave_IC.ll @@ -1,5 +1,5 @@ -; RUN: opt < %s -passes=loop-vectorize -S -mcpu=pwr9 -interleave-small-loop-scalar-reduction=true 2>&1 | FileCheck %s -; RUN: opt < %s -passes='loop-vectorize' -S -mcpu=pwr9 -interleave-small-loop-scalar-reduction=true 2>&1 | FileCheck %s +; RUN: opt < %s -passes=loop-vectorize -S -mcpu=pwr9 2>&1 | FileCheck %s +; RUN: opt < %s -passes='loop-vectorize' -S -mcpu=pwr9 2>&1 | FileCheck %s ; CHECK-LABEL: vector.body ; CHECK: load double, ptr diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll index da6dc34..72d9691 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -36,10 +36,10 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %i.0 = add nsw i32 %i.0.in8, -1 ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %idxprom = zext i32 %i.0 to i64 ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %arrayidx = getelementptr inbounds i32, ptr %B, i64 %idxprom -; CHECK-NEXT: LV: Found an estimated cost of 11 for VF vscale x 4 For instruction: %1 = load i32, ptr %arrayidx, align 4 +; CHECK-NEXT: LV: Found an estimated cost of 13 for VF vscale x 4 For instruction: %1 = load i32, ptr %arrayidx, align 4 ; CHECK-NEXT: LV: Found an estimated cost of 2 for VF vscale x 4 For instruction: %add9 = add i32 %1, 1 ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %arrayidx3 = getelementptr inbounds i32, ptr %A, i64 %idxprom -; CHECK-NEXT: LV: Found an estimated cost of 11 for VF vscale x 4 For instruction: store i32 %add9, ptr %arrayidx3, align 4 +; CHECK-NEXT: LV: Found an estimated cost of 13 for VF vscale x 4 For instruction: store i32 %add9, ptr %arrayidx3, align 4 ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %cmp = icmp ugt i64 %indvars.iv, 1 ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !0 @@ -86,10 +86,10 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %i.0 = add nsw i32 %i.0.in8, -1 ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %idxprom = zext i32 %i.0 to i64 ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %arrayidx = getelementptr inbounds i32, ptr %B, i64 %idxprom -; CHECK-NEXT: LV: Found an estimated cost of 11 for VF vscale x 4 For instruction: %1 = load i32, ptr %arrayidx, align 4 +; CHECK-NEXT: LV: Found an estimated cost of 13 for VF vscale x 4 For instruction: %1 = load i32, ptr %arrayidx, align 4 ; CHECK-NEXT: LV: Found an estimated cost of 2 for VF vscale x 4 For instruction: %add9 = add i32 %1, 1 ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %arrayidx3 = getelementptr inbounds i32, ptr %A, i64 %idxprom -; CHECK-NEXT: LV: Found an estimated cost of 11 for VF vscale x 4 For instruction: store i32 %add9, ptr %arrayidx3, align 4 +; CHECK-NEXT: LV: Found an estimated cost of 13 for VF vscale x 4 For instruction: store i32 %add9, ptr %arrayidx3, align 4 ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %cmp = icmp ugt i64 %indvars.iv, 1 ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !0 @@ -112,7 +112,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers ; CHECK-NEXT: LV: The target has 31 registers of RISCV::GPRRC register class ; CHECK-NEXT: LV: The target has 32 registers of RISCV::VRRC register class -; CHECK-NEXT: LV: Loop cost is 28 +; CHECK-NEXT: LV: Loop cost is 32 ; CHECK-NEXT: LV: IC is 1 ; CHECK-NEXT: LV: VF is vscale x 4 ; CHECK-NEXT: LV: Not Interleaving. @@ -122,6 +122,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: Executing best plan with VF=vscale x 4, UF=1 ; CHECK: LV: Interleaving disabled by the pass manager ; CHECK-NEXT: LV: Vectorizing: innermost loop. +; CHECK-EMPTY: ; entry: %cmp7 = icmp sgt i32 %n, 0 @@ -176,10 +177,10 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %i.0 = add nsw i32 %i.0.in8, -1 ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %idxprom = zext i32 %i.0 to i64 ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %arrayidx = getelementptr inbounds float, ptr %B, i64 %idxprom -; CHECK-NEXT: LV: Found an estimated cost of 11 for VF vscale x 4 For instruction: %1 = load float, ptr %arrayidx, align 4 +; CHECK-NEXT: LV: Found an estimated cost of 13 for VF vscale x 4 For instruction: %1 = load float, ptr %arrayidx, align 4 ; CHECK-NEXT: LV: Found an estimated cost of 2 for VF vscale x 4 For instruction: %conv1 = fadd float %1, 1.000000e+00 ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %arrayidx3 = getelementptr inbounds float, ptr %A, i64 %idxprom -; CHECK-NEXT: LV: Found an estimated cost of 11 for VF vscale x 4 For instruction: store float %conv1, ptr %arrayidx3, align 4 +; CHECK-NEXT: LV: Found an estimated cost of 13 for VF vscale x 4 For instruction: store float %conv1, ptr %arrayidx3, align 4 ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %cmp = icmp ugt i64 %indvars.iv, 1 ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !0 @@ -226,10 +227,10 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %i.0 = add nsw i32 %i.0.in8, -1 ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %idxprom = zext i32 %i.0 to i64 ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %arrayidx = getelementptr inbounds float, ptr %B, i64 %idxprom -; CHECK-NEXT: LV: Found an estimated cost of 11 for VF vscale x 4 For instruction: %1 = load float, ptr %arrayidx, align 4 +; CHECK-NEXT: LV: Found an estimated cost of 13 for VF vscale x 4 For instruction: %1 = load float, ptr %arrayidx, align 4 ; CHECK-NEXT: LV: Found an estimated cost of 2 for VF vscale x 4 For instruction: %conv1 = fadd float %1, 1.000000e+00 ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %arrayidx3 = getelementptr inbounds float, ptr %A, i64 %idxprom -; CHECK-NEXT: LV: Found an estimated cost of 11 for VF vscale x 4 For instruction: store float %conv1, ptr %arrayidx3, align 4 +; CHECK-NEXT: LV: Found an estimated cost of 13 for VF vscale x 4 For instruction: store float %conv1, ptr %arrayidx3, align 4 ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %cmp = icmp ugt i64 %indvars.iv, 1 ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !0 @@ -252,7 +253,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers ; CHECK-NEXT: LV: The target has 31 registers of RISCV::GPRRC register class ; CHECK-NEXT: LV: The target has 32 registers of RISCV::VRRC register class -; CHECK-NEXT: LV: Loop cost is 28 +; CHECK-NEXT: LV: Loop cost is 32 ; CHECK-NEXT: LV: IC is 1 ; CHECK-NEXT: LV: VF is vscale x 4 ; CHECK-NEXT: LV: Not Interleaving. diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/global_remove_same.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/global_remove_same.ll new file mode 100644 index 0000000..d3d13ae --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/global_remove_same.ll @@ -0,0 +1,15 @@ +; RUN: opt -S < %s | FileCheck %s + +define i32 @foo() { +; CHECK-LABEL: @foo( +; CHECK-NEXT: [[RESULT:%.*]] = call i32 @bar(i32 0, i32 1) +; CHECK-NEXT: ret i32 [[RESULT]] +; + %result = call i32 @bar(i32 0, i32 1) + ret i32 %result +} + +declare i32 @bar(i32, i32) +; CHECK-LABEL: @bar( +; CHECK-SAME: i32 +; CHECK-SAME: i32 diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/global_remove_same.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/global_remove_same.ll.expected new file mode 100644 index 0000000..e76efae --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/global_remove_same.ll.expected @@ -0,0 +1,13 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S < %s | FileCheck %s + +define i32 @foo() { +; CHECK-LABEL: @foo( +; CHECK-NEXT: [[RESULT:%.*]] = call i32 @bar(i32 0, i32 1) +; CHECK-NEXT: ret i32 [[RESULT]] +; + %result = call i32 @bar(i32 0, i32 1) + ret i32 %result +} + +declare i32 @bar(i32, i32) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/global_remove_same.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/global_remove_same.test new file mode 100644 index 0000000..5d447bab --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/global_remove_same.test @@ -0,0 +1,4 @@ +## Basic test checking global checks split over multiple lines are removed together +# RUN: cp -f %S/Inputs/global_remove_same.ll %t.ll && %update_test_checks %t.ll +# RUN: diff -u %t.ll %S/Inputs/global_remove_same.ll.expected + diff --git a/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp b/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp index 2b438a8..2bfc970 100644 --- a/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp +++ b/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp @@ -286,6 +286,8 @@ static opt<bool> Verify("verify", desc("Verify the DWARF debug info."), cat(DwarfDumpCategory)); static opt<ErrorDetailLevel> ErrorDetails( "error-display", init(Unspecified), + desc("Set the level of detail and summary to display when verifying " + "(implies --verify)"), values(clEnumValN(NoDetailsOrSummary, "quiet", "Only display whether errors occurred."), clEnumValN(NoDetailsOnlySummary, "summary", @@ -295,6 +297,11 @@ static opt<ErrorDetailLevel> ErrorDetails( clEnumValN(BothDetailsAndSummary, "full", "Display each error as well as a summary. [default]")), cat(DwarfDumpCategory)); +static opt<std::string> JsonErrSummaryFile( + "verify-json", init(""), + desc("Output JSON-formatted error summary to the specified file. " + "(Implies --verify)"), + value_desc("filename.json"), cat(DwarfDumpCategory)); static opt<bool> Quiet("quiet", desc("Use with -verify to not emit to STDOUT."), cat(DwarfDumpCategory)); static opt<bool> DumpUUID("uuid", desc("Show the UUID for each architecture."), @@ -349,6 +356,7 @@ static DIDumpOptions getDumpOpts(DWARFContext &C) { ErrorDetails != NoDetailsOrSummary; DumpOpts.ShowAggregateErrors = ErrorDetails != OnlyDetailsNoSummary && ErrorDetails != NoDetailsOnlySummary; + DumpOpts.JsonErrSummaryFile = JsonErrSummaryFile; return DumpOpts.noImplicitRecursion(); } return DumpOpts; @@ -834,8 +842,10 @@ int main(int argc, char **argv) { "-verbose is currently not supported"; return 1; } - if (!Verify && ErrorDetails != Unspecified) - WithColor::warning() << "-error-detail has no affect without -verify"; + // -error-detail and -json-summary-file both imply -verify + if (ErrorDetails != Unspecified || !JsonErrSummaryFile.empty()) { + Verify = true; + } std::error_code EC; ToolOutputFile OutputFile(OutputFilename, EC, sys::fs::OF_TextWithCRLF); diff --git a/llvm/unittests/ADT/APFloatTest.cpp b/llvm/unittests/ADT/APFloatTest.cpp index baf055e..6e4dda8 100644 --- a/llvm/unittests/ADT/APFloatTest.cpp +++ b/llvm/unittests/ADT/APFloatTest.cpp @@ -578,6 +578,11 @@ TEST(APFloatTest, MinNum) { EXPECT_EQ(1.0, minnum(f2, f1).convertToDouble()); EXPECT_EQ(1.0, minnum(f1, nan).convertToDouble()); EXPECT_EQ(1.0, minnum(nan, f1).convertToDouble()); + + APFloat zp(0.0); + APFloat zn(-0.0); + EXPECT_EQ(-0.0, minnum(zp, zn).convertToDouble()); + EXPECT_EQ(-0.0, minnum(zn, zp).convertToDouble()); } TEST(APFloatTest, MaxNum) { @@ -589,6 +594,11 @@ TEST(APFloatTest, MaxNum) { EXPECT_EQ(2.0, maxnum(f2, f1).convertToDouble()); EXPECT_EQ(1.0, maxnum(f1, nan).convertToDouble()); EXPECT_EQ(1.0, maxnum(nan, f1).convertToDouble()); + + APFloat zp(0.0); + APFloat zn(-0.0); + EXPECT_EQ(0.0, maxnum(zp, zn).convertToDouble()); + EXPECT_EQ(0.0, maxnum(zn, zp).convertToDouble()); } TEST(APFloatTest, Minimum) { diff --git a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp index bb8e76a..e077268 100644 --- a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp +++ b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp @@ -6,11 +6,13 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Analysis/MemoryLocation.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/AsmParser/Parser.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/TargetLowering.h" +#include "llvm/IR/MDBuilder.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/Support/KnownBits.h" #include "llvm/Support/SourceMgr.h" @@ -728,4 +730,70 @@ TEST_F(AArch64SelectionDAGTest, ReplaceAllUsesWith) { EXPECT_EQ(DAG->getPCSections(New.getNode()), MD); } +TEST_F(AArch64SelectionDAGTest, computeKnownBits_extload_known01) { + SDLoc Loc; + auto Int8VT = EVT::getIntegerVT(Context, 8); + auto Int32VT = EVT::getIntegerVT(Context, 32); + auto Int64VT = EVT::getIntegerVT(Context, 64); + auto Ptr = DAG->getConstant(0, Loc, Int64VT); + auto PtrInfo = + MachinePointerInfo::getFixedStack(DAG->getMachineFunction(), 0); + AAMDNodes AA; + MDBuilder MDHelper(*DAG->getContext()); + MDNode *Range = MDHelper.createRange(APInt(8, 0), APInt(8, 2)); + MachineMemOperand *MMO = DAG->getMachineFunction().getMachineMemOperand( + PtrInfo, MachineMemOperand::MOLoad, 8, Align(8), AA, Range); + + auto ALoad = DAG->getExtLoad(ISD::EXTLOAD, Loc, Int32VT, DAG->getEntryNode(), + Ptr, Int8VT, MMO); + KnownBits Known = DAG->computeKnownBits(ALoad); + EXPECT_EQ(Known.Zero, APInt(32, 0xfe)); + EXPECT_EQ(Known.One, APInt(32, 0)); + + auto ZLoad = DAG->getExtLoad(ISD::ZEXTLOAD, Loc, Int32VT, DAG->getEntryNode(), + Ptr, Int8VT, MMO); + Known = DAG->computeKnownBits(ZLoad); + EXPECT_EQ(Known.Zero, APInt(32, 0xfffffffe)); + EXPECT_EQ(Known.One, APInt(32, 0)); + + auto SLoad = DAG->getExtLoad(ISD::SEXTLOAD, Loc, Int32VT, DAG->getEntryNode(), + Ptr, Int8VT, MMO); + Known = DAG->computeKnownBits(SLoad); + EXPECT_EQ(Known.Zero, APInt(32, 0xfffffffe)); + EXPECT_EQ(Known.One, APInt(32, 0)); +} + +TEST_F(AArch64SelectionDAGTest, computeKnownBits_extload_knownnegative) { + SDLoc Loc; + auto Int8VT = EVT::getIntegerVT(Context, 8); + auto Int32VT = EVT::getIntegerVT(Context, 32); + auto Int64VT = EVT::getIntegerVT(Context, 64); + auto Ptr = DAG->getConstant(0, Loc, Int64VT); + auto PtrInfo = + MachinePointerInfo::getFixedStack(DAG->getMachineFunction(), 0); + AAMDNodes AA; + MDBuilder MDHelper(*DAG->getContext()); + MDNode *Range = MDHelper.createRange(APInt(8, 0xf0), APInt(8, 0xff)); + MachineMemOperand *MMO = DAG->getMachineFunction().getMachineMemOperand( + PtrInfo, MachineMemOperand::MOLoad, 8, Align(8), AA, Range); + + auto ALoad = DAG->getExtLoad(ISD::EXTLOAD, Loc, Int32VT, DAG->getEntryNode(), + Ptr, Int8VT, MMO); + KnownBits Known = DAG->computeKnownBits(ALoad); + EXPECT_EQ(Known.Zero, APInt(32, 0)); + EXPECT_EQ(Known.One, APInt(32, 0xf0)); + + auto ZLoad = DAG->getExtLoad(ISD::ZEXTLOAD, Loc, Int32VT, DAG->getEntryNode(), + Ptr, Int8VT, MMO); + Known = DAG->computeKnownBits(ZLoad); + EXPECT_EQ(Known.Zero, APInt(32, 0xffffff00)); + EXPECT_EQ(Known.One, APInt(32, 0x000000f0)); + + auto SLoad = DAG->getExtLoad(ISD::SEXTLOAD, Loc, Int32VT, DAG->getEntryNode(), + Ptr, Int8VT, MMO); + Known = DAG->computeKnownBits(SLoad); + EXPECT_EQ(Known.Zero, APInt(32, 0)); + EXPECT_EQ(Known.One, APInt(32, 0xfffffff0)); +} + } // end namespace llvm diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp index 31e3b9d..b2cd71e 100644 --- a/llvm/unittests/IR/VerifierTest.cpp +++ b/llvm/unittests/IR/VerifierTest.cpp @@ -339,5 +339,33 @@ TEST(VerifierTest, SwitchInst) { EXPECT_TRUE(verifyFunction(*F)); } +TEST(VerifierTest, CrossFunctionRef) { + LLVMContext C; + Module M("M", C); + FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false); + Function *F1 = Function::Create(FTy, Function::ExternalLinkage, "foo1", M); + Function *F2 = Function::Create(FTy, Function::ExternalLinkage, "foo2", M); + BasicBlock *Entry1 = BasicBlock::Create(C, "entry", F1); + BasicBlock *Entry2 = BasicBlock::Create(C, "entry", F2); + Type *I32 = Type::getInt32Ty(C); + + Value *Alloca = new AllocaInst(I32, 0, "alloca", Entry1); + ReturnInst::Create(C, Entry1); + + Instruction *Store = new StoreInst(ConstantInt::get(I32, 0), Alloca, Entry2); + ReturnInst::Create(C, Entry2); + + std::string Error; + raw_string_ostream ErrorOS(Error); + EXPECT_TRUE(verifyModule(M, &ErrorOS)); + EXPECT_TRUE( + StringRef(ErrorOS.str()) + .starts_with("Referring to an instruction in another function!")); + + // Explicitly erase the store to avoid a use-after-free when the module is + // destroyed. + Store->eraseFromParent(); +} + } // end anonymous namespace } // end namespace llvm diff --git a/llvm/unittests/Support/RISCVISAInfoTest.cpp b/llvm/unittests/Support/RISCVISAInfoTest.cpp index df4c7f7..82cce23 100644 --- a/llvm/unittests/Support/RISCVISAInfoTest.cpp +++ b/llvm/unittests/Support/RISCVISAInfoTest.cpp @@ -752,6 +752,7 @@ R"(All available -march extensions for RISC-V zmmul 1.0 za128rs 1.0 za64rs 1.0 + zacas 1.0 zawrs 1.0 zfa 1.0 zfh 1.0 @@ -873,7 +874,6 @@ Experimental extensions zimop 0.1 zaamo 0.2 zabha 1.0 - zacas 1.0 zalasr 0.1 zalrsc 0.2 zfbfmin 1.0 diff --git a/llvm/unittests/Target/ARM/MachineInstrTest.cpp b/llvm/unittests/Target/ARM/MachineInstrTest.cpp index 3a76054..aeb25bf 100644 --- a/llvm/unittests/Target/ARM/MachineInstrTest.cpp +++ b/llvm/unittests/Target/ARM/MachineInstrTest.cpp @@ -1126,9 +1126,7 @@ TEST(MachineInstr, HasSideEffects) { VLDR_VPR_post, VLDR_VPR_pre, VLLDM, - VLLDM_T2, VLSTM, - VLSTM_T2, VMRS, VMRS_FPCXTNS, VMRS_FPCXTS, diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp index d47df59..fc958f5 100644 --- a/llvm/utils/TableGen/DXILEmitter.cpp +++ b/llvm/utils/TableGen/DXILEmitter.cpp @@ -11,11 +11,14 @@ // //===----------------------------------------------------------------------===// +#include "CodeGenTarget.h" #include "SequenceToOffsetTable.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringSet.h" #include "llvm/ADT/StringSwitch.h" +#include "llvm/CodeGenTypes/MachineValueType.h" #include "llvm/Support/DXILABI.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" @@ -30,28 +33,15 @@ struct DXILShaderModel { int Minor = 0; }; -struct DXILParameter { - int Pos; // position in parameter list - ParameterKind Kind; - StringRef Name; // short, unique name - StringRef Doc; // the documentation description of this parameter - bool IsConst; // whether this argument requires a constant value in the IR - StringRef EnumName; // the name of the enum type if applicable - int MaxValue; // the maximum value for this parameter if applicable - DXILParameter(const Record *R); -}; - struct DXILOperationDesc { - StringRef OpName; // name of DXIL operation + std::string OpName; // name of DXIL operation int OpCode; // ID of DXIL operation StringRef OpClass; // name of the opcode class - StringRef Category; // classification for this instruction StringRef Doc; // the documentation description of this instruction - - SmallVector<DXILParameter> Params; // the operands that this instruction takes - SmallVector<ParameterKind> OverloadTypes; // overload types if applicable - StringRef Attr; // operation attribute; reference to string representation - // of llvm::Attribute::AttrKind + SmallVector<MVT::SimpleValueType> OpTypes; // Vector of operand types - + // return type is at index 0 + SmallVector<std::string> + OpAttributes; // operation attribute represented as strings StringRef Intrinsic; // The llvm intrinsic map to OpName. Default is "" which // means no map exists bool IsDeriv = false; // whether this is some kind of derivative @@ -74,81 +64,99 @@ struct DXILOperationDesc { }; } // end anonymous namespace -/*! - Convert DXIL type name string to dxil::ParameterKind - - @param typeNameStr Type name string - @return ParameterKind As defined in llvm/Support/DXILABI.h -*/ -static ParameterKind lookupParameterKind(StringRef typeNameStr) { - auto paramKind = StringSwitch<ParameterKind>(typeNameStr) - .Case("llvm_void_ty", ParameterKind::VOID) - .Case("llvm_half_ty", ParameterKind::HALF) - .Case("llvm_float_ty", ParameterKind::FLOAT) - .Case("llvm_double_ty", ParameterKind::DOUBLE) - .Case("llvm_i1_ty", ParameterKind::I1) - .Case("llvm_i8_ty", ParameterKind::I8) - .Case("llvm_i16_ty", ParameterKind::I16) - .Case("llvm_i32_ty", ParameterKind::I32) - .Case("llvm_i64_ty", ParameterKind::I64) - .Case("llvm_anyfloat_ty", ParameterKind::OVERLOAD) - .Case("llvm_anyint_ty", ParameterKind::OVERLOAD) - .Case("dxil_handle_ty", ParameterKind::DXIL_HANDLE) - .Case("dxil_cbuffer_ty", ParameterKind::CBUFFER_RET) - .Case("dxil_resource_ty", ParameterKind::RESOURCE_RET) - .Default(ParameterKind::INVALID); - assert(paramKind != ParameterKind::INVALID && - "Unsupported DXIL Type specified"); - return paramKind; +/// Convert DXIL type name string to dxil::ParameterKind +/// +/// \param VT Simple Value Type +/// \return ParameterKind As defined in llvm/Support/DXILABI.h + +static ParameterKind getParameterKind(MVT::SimpleValueType VT) { + switch (VT) { + case MVT::isVoid: + return ParameterKind::VOID; + case MVT::f16: + return ParameterKind::HALF; + case MVT::f32: + return ParameterKind::FLOAT; + case MVT::f64: + return ParameterKind::DOUBLE; + case MVT::i1: + return ParameterKind::I1; + case MVT::i8: + return ParameterKind::I8; + case MVT::i16: + return ParameterKind::I16; + case MVT::i32: + return ParameterKind::I32; + case MVT::fAny: + case MVT::iAny: + return ParameterKind::OVERLOAD; + default: + llvm_unreachable("Support for specified DXIL Type not yet implemented"); + } } +/// Construct an object using the DXIL Operation records specified +/// in DXIL.td. This serves as the single source of reference of +/// the information extracted from the specified Record R, for +/// C++ code generated by this TableGen backend. +// \param R Object representing TableGen record of a DXIL Operation DXILOperationDesc::DXILOperationDesc(const Record *R) { - OpName = R->getValueAsString("OpName"); + OpName = R->getNameInitAsString(); OpCode = R->getValueAsInt("OpCode"); - OpClass = R->getValueAsDef("OpClass")->getValueAsString("Name"); - Category = R->getValueAsDef("OpCategory")->getValueAsString("Name"); - if (R->getValue("llvm_intrinsic")) { - auto *IntrinsicDef = R->getValueAsDef("llvm_intrinsic"); + Doc = R->getValueAsString("Doc"); + + if (R->getValue("LLVMIntrinsic")) { + auto *IntrinsicDef = R->getValueAsDef("LLVMIntrinsic"); auto DefName = IntrinsicDef->getName(); assert(DefName.starts_with("int_") && "invalid intrinsic name"); // Remove the int_ from intrinsic name. Intrinsic = DefName.substr(4); + // TODO: It is expected that return type and parameter types of + // DXIL Operation are the same as that of the intrinsic. Deviations + // are expected to be encoded in TableGen record specification and + // handled accordingly here. Support to be added later, as needed. + // Get parameter type list of the intrinsic. Types attribute contains + // the list of as [returnType, param1Type,, param2Type, ...] + + OverloadParamIndex = -1; + auto TypeRecs = IntrinsicDef->getValueAsListOfDefs("Types"); + unsigned TypeRecsSize = TypeRecs.size(); + // Populate return type and parameter type names + for (unsigned i = 0; i < TypeRecsSize; i++) { + auto TR = TypeRecs[i]; + OpTypes.emplace_back(getValueType(TR->getValueAsDef("VT"))); + // Get the overload parameter index. + // TODO : Seems hacky. Is it possible that more than one parameter can + // be of overload kind?? + // TODO: Check for any additional constraints specified for DXIL operation + // restricting return type. + if (i > 0) { + auto &CurParam = OpTypes.back(); + if (getParameterKind(CurParam) >= ParameterKind::OVERLOAD) { + OverloadParamIndex = i; + } + } + } + // Get the operation class + OpClass = R->getValueAsDef("OpClass")->getName(); + + // NOTE: For now, assume that attributes of DXIL Operation are the same as + // that of the intrinsic. Deviations are expected to be encoded in TableGen + // record specification and handled accordingly here. Support to be added + // later. + auto IntrPropList = IntrinsicDef->getValueAsListInit("IntrProperties"); + auto IntrPropListSize = IntrPropList->size(); + for (unsigned i = 0; i < IntrPropListSize; i++) { + OpAttributes.emplace_back(IntrPropList->getElement(i)->getAsString()); + } } - - Doc = R->getValueAsString("Doc"); - - ListInit *ParamList = R->getValueAsListInit("Params"); - OverloadParamIndex = -1; - for (unsigned I = 0; I < ParamList->size(); ++I) { - Record *Param = ParamList->getElementAsRecord(I); - Params.emplace_back(DXILParameter(Param)); - auto &CurParam = Params.back(); - if (CurParam.Kind >= ParameterKind::OVERLOAD) - OverloadParamIndex = I; - } - ListInit *OverloadTypeList = R->getValueAsListInit("OverloadTypes"); - - for (unsigned I = 0; I < OverloadTypeList->size(); ++I) { - Record *R = OverloadTypeList->getElementAsRecord(I); - OverloadTypes.emplace_back(lookupParameterKind(R->getNameInitAsString())); - } - Attr = StringRef(R->getValue("Attribute")->getNameInitAsString()); } -DXILParameter::DXILParameter(const Record *R) { - Name = R->getValueAsString("Name"); - Pos = R->getValueAsInt("Pos"); - Kind = - lookupParameterKind(R->getValue("ParamType")->getValue()->getAsString()); - if (R->getValue("Doc")) - Doc = R->getValueAsString("Doc"); - IsConst = R->getValueAsBit("IsConstant"); - EnumName = R->getValueAsString("EnumName"); - MaxValue = R->getValueAsInt("MaxValue"); -} - -static std::string parameterKindToString(ParameterKind Kind) { +/// Return a string representation of ParameterKind enum +/// \param Kind Parameter Kind enum value +/// \return std::string string representation of input Kind +static std::string getParameterKindStr(ParameterKind Kind) { switch (Kind) { case ParameterKind::INVALID: return "INVALID"; @@ -182,92 +190,77 @@ static std::string parameterKindToString(ParameterKind Kind) { llvm_unreachable("Unknown llvm::dxil::ParameterKind enum"); } -static void emitDXILOpEnum(DXILOperationDesc &Op, raw_ostream &OS) { - // Name = ID, // Doc - OS << Op.OpName << " = " << Op.OpCode << ", // " << Op.Doc << "\n"; -} +/// Return a string representation of OverloadKind enum that maps to +/// input Simple Value Type enum +/// \param VT Simple Value Type enum +/// \return std::string string representation of OverloadKind -static std::string buildCategoryStr(StringSet<> &Cetegorys) { - std::string Str; - raw_string_ostream OS(Str); - for (auto &It : Cetegorys) { - OS << " " << It.getKey(); +static std::string getOverloadKindStr(MVT::SimpleValueType VT) { + switch (VT) { + case MVT::isVoid: + return "OverloadKind::VOID"; + case MVT::f16: + return "OverloadKind::HALF"; + case MVT::f32: + return "OverloadKind::FLOAT"; + case MVT::f64: + return "OverloadKind::DOUBLE"; + case MVT::i1: + return "OverloadKind::I1"; + case MVT::i8: + return "OverloadKind::I8"; + case MVT::i16: + return "OverloadKind::I16"; + case MVT::i32: + return "OverloadKind::I32"; + case MVT::i64: + return "OverloadKind::I64"; + case MVT::iAny: + return "OverloadKind::I16 | OverloadKind::I32 | OverloadKind::I64"; + case MVT::fAny: + return "OverloadKind::HALF | OverloadKind::FLOAT | OverloadKind::DOUBLE"; + default: + llvm_unreachable( + "Support for specified parameter OverloadKind not yet implemented"); } - return OS.str(); } -// Emit enum declaration for DXIL. +/// Emit Enums of DXIL Ops +/// \param A vector of DXIL Ops +/// \param Output stream static void emitDXILEnums(std::vector<DXILOperationDesc> &Ops, raw_ostream &OS) { - // Sort by Category + OpName. + // Sort by OpCode llvm::sort(Ops, [](DXILOperationDesc &A, DXILOperationDesc &B) { - // Group by Category first. - if (A.Category == B.Category) - // Inside same Category, order by OpName. - return A.OpName < B.OpName; - else - return A.Category < B.Category; + return A.OpCode < B.OpCode; }); OS << "// Enumeration for operations specified by DXIL\n"; OS << "enum class OpCode : unsigned {\n"; - StringMap<StringSet<>> ClassMap; - StringRef PrevCategory = ""; for (auto &Op : Ops) { - StringRef Category = Op.Category; - if (Category != PrevCategory) { - OS << "\n// " << Category << "\n"; - PrevCategory = Category; - } - emitDXILOpEnum(Op, OS); - auto It = ClassMap.find(Op.OpClass); - if (It != ClassMap.end()) { - It->second.insert(Op.Category); - } else { - ClassMap[Op.OpClass].insert(Op.Category); - } + // Name = ID, // Doc + OS << Op.OpName << " = " << Op.OpCode << ", // " << Op.Doc << "\n"; } OS << "\n};\n\n"; - std::vector<std::pair<std::string, std::string>> ClassVec; - for (auto &It : ClassMap) { - ClassVec.emplace_back( - std::pair(It.getKey().str(), buildCategoryStr(It.second))); - } - // Sort by Category + ClassName. - llvm::sort(ClassVec, [](std::pair<std::string, std::string> &A, - std::pair<std::string, std::string> &B) { - StringRef ClassA = A.first; - StringRef CategoryA = A.second; - StringRef ClassB = B.first; - StringRef CategoryB = B.second; - // Group by Category first. - if (CategoryA == CategoryB) - // Inside same Category, order by ClassName. - return ClassA < ClassB; - else - return CategoryA < CategoryB; - }); - OS << "// Groups for DXIL operations with equivalent function templates\n"; OS << "enum class OpCodeClass : unsigned {\n"; - PrevCategory = ""; - for (auto &It : ClassVec) { - - StringRef Category = It.second; - if (Category != PrevCategory) { - OS << "\n// " << Category << "\n"; - PrevCategory = Category; - } - StringRef Name = It.first; - OS << Name << ",\n"; + // Build an OpClass set to print + SmallSet<StringRef, 2> OpClassSet; + for (auto &Op : Ops) { + OpClassSet.insert(Op.OpClass); + } + for (auto &C : OpClassSet) { + OS << C << ",\n"; } OS << "\n};\n\n"; } -// Emit map from llvm intrinsic to DXIL operation. +/// Emit map of DXIL operation to LLVM or DirectX intrinsic +/// \param A vector of DXIL Ops +/// \param Output stream static void emitDXILIntrinsicMap(std::vector<DXILOperationDesc> &Ops, raw_ostream &OS) { OS << "\n"; @@ -285,75 +278,27 @@ static void emitDXILIntrinsicMap(std::vector<DXILOperationDesc> &Ops, OS << "\n"; } -/*! - Convert operation attribute string to Attribute enum - - @param Attr string reference - @return std::string Attribute enum string - */ -static std::string emitDXILOperationAttr(StringRef Attr) { - return StringSwitch<std::string>(Attr) - .Case("ReadNone", "Attribute::ReadNone") - .Case("ReadOnly", "Attribute::ReadOnly") - .Default("Attribute::None"); -} - -static std::string overloadKindStr(ParameterKind Overload) { - switch (Overload) { - case ParameterKind::HALF: - return "OverloadKind::HALF"; - case ParameterKind::FLOAT: - return "OverloadKind::FLOAT"; - case ParameterKind::DOUBLE: - return "OverloadKind::DOUBLE"; - case ParameterKind::I1: - return "OverloadKind::I1"; - case ParameterKind::I8: - return "OverloadKind::I8"; - case ParameterKind::I16: - return "OverloadKind::I16"; - case ParameterKind::I32: - return "OverloadKind::I32"; - case ParameterKind::I64: - return "OverloadKind::I64"; - case ParameterKind::VOID: - return "OverloadKind::VOID"; - default: - return "OverloadKind::UNKNOWN"; - } -} - -static std::string -getDXILOperationOverloads(SmallVector<ParameterKind> Overloads) { - // Format is: OverloadKind::FLOAT | OverloadKind::HALF - auto It = Overloads.begin(); - std::string Result; - raw_string_ostream OS(Result); - OS << overloadKindStr(*It); - for (++It; It != Overloads.end(); ++It) { - OS << " | " << overloadKindStr(*It); +/// Convert operation attribute string to Attribute enum +/// +/// \param Attr string reference +/// \return std::string Attribute enum string + +static std::string emitDXILOperationAttr(SmallVector<std::string> Attrs) { + for (auto Attr : Attrs) { + // TODO: For now just recognize IntrNoMem and IntrReadMem as valid and + // ignore others. + if (Attr == "IntrNoMem") { + return "Attribute::ReadNone"; + } else if (Attr == "IntrReadMem") { + return "Attribute::ReadOnly"; + } } - return OS.str(); -} - -static std::string lowerFirstLetter(StringRef Name) { - if (Name.empty()) - return ""; - - std::string LowerName = Name.str(); - LowerName[0] = llvm::toLower(Name[0]); - return LowerName; -} - -static std::string getDXILOpClassName(StringRef OpClass) { - // Lower first letter expect for special case. - return StringSwitch<std::string>(OpClass) - .Case("CBufferLoad", "cbufferLoad") - .Case("CBufferLoadLegacy", "cbufferLoadLegacy") - .Case("GSInstanceID", "gsInstanceID") - .Default(lowerFirstLetter(OpClass)); + return "Attribute::None"; } +/// Emit DXIL operation table +/// \param A vector of DXIL Ops +/// \param Output stream static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops, raw_ostream &OS) { // Sort by OpCode. @@ -369,15 +314,16 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops, StringMap<SmallVector<ParameterKind>> ParameterMap; StringSet<> ClassSet; for (auto &Op : Ops) { - OpStrings.add(Op.OpName.str()); + OpStrings.add(Op.OpName); if (ClassSet.contains(Op.OpClass)) continue; ClassSet.insert(Op.OpClass); - OpClassStrings.add(getDXILOpClassName(Op.OpClass)); + OpClassStrings.add(Op.OpClass.data()); SmallVector<ParameterKind> ParamKindVec; - for (auto &Param : Op.Params) { - ParamKindVec.emplace_back(Param.Kind); + // ParamKindVec is a vector of parameters. Skip return type at index 0 + for (unsigned i = 1; i < Op.OpTypes.size(); i++) { + ParamKindVec.emplace_back(getParameterKind(Op.OpTypes[i])); } ParameterMap[Op.OpClass] = ParamKindVec; Parameters.add(ParamKindVec); @@ -389,7 +335,7 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops, Parameters.layout(); // Emit the DXIL operation table. - //{dxil::OpCode::Sin, OpCodeNameIndex, OpCodeClass::Unary, + //{dxil::OpCode::Sin, OpCodeNameIndex, OpCodeClass::unary, // OpCodeClassNameIndex, // OverloadKind::FLOAT | OverloadKind::HALF, Attribute::AttrKind::ReadNone, 0, // 3, ParameterTableOffset}, @@ -398,12 +344,12 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops, OS << " static const OpCodeProperty OpCodeProps[] = {\n"; for (auto &Op : Ops) { - OS << " { dxil::OpCode::" << Op.OpName << ", " - << OpStrings.get(Op.OpName.str()) << ", OpCodeClass::" << Op.OpClass - << ", " << OpClassStrings.get(getDXILOpClassName(Op.OpClass)) << ", " - << getDXILOperationOverloads(Op.OverloadTypes) << ", " - << emitDXILOperationAttr(Op.Attr) << ", " << Op.OverloadParamIndex - << ", " << Op.Params.size() << ", " + OS << " { dxil::OpCode::" << Op.OpName << ", " << OpStrings.get(Op.OpName) + << ", OpCodeClass::" << Op.OpClass << ", " + << OpClassStrings.get(Op.OpClass.data()) << ", " + << getOverloadKindStr(Op.OpTypes[0]) << ", " + << emitDXILOperationAttr(Op.OpAttributes) << ", " + << Op.OverloadParamIndex << ", " << Op.OpTypes.size() - 1 << ", " << Parameters.get(ParameterMap[Op.OpClass]) << " },\n"; } OS << " };\n"; @@ -418,7 +364,7 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops, "OpCodeProperty &B) {\n"; OS << " return A.OpCode < B.OpCode;\n"; OS << " });\n"; - OS << " assert(Prop && \"fail to find OpCodeProperty\");\n"; + OS << " assert(Prop && \"failed to find OpCodeProperty\");\n"; OS << " return Prop;\n"; OS << "}\n\n"; @@ -450,7 +396,7 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops, Parameters.emit( OS, [](raw_ostream &ParamOS, ParameterKind Kind) { - ParamOS << "ParameterKind::" << parameterKindToString(Kind); + ParamOS << "ParameterKind::" << getParameterKindStr(Kind); }, "ParameterKind::INVALID"); OS << " };\n\n"; @@ -459,30 +405,28 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops, OS << "}\n "; } +/// Entry function call that invokes the functionality of this TableGen backend +/// \param Records TableGen records of DXIL Operations defined in DXIL.td +/// \param OS output stream static void EmitDXILOperation(RecordKeeper &Records, raw_ostream &OS) { - std::vector<Record *> Ops = Records.getAllDerivedDefinitions("DXILOperation"); OS << "// Generated code, do not edit.\n"; OS << "\n"; - + // Get all DXIL Ops to intrinsic mapping records + std::vector<Record *> OpIntrMaps = + Records.getAllDerivedDefinitions("DXILOpMapping"); std::vector<DXILOperationDesc> DXILOps; - DXILOps.reserve(Ops.size()); - for (auto *Record : Ops) { + for (auto *Record : OpIntrMaps) { DXILOps.emplace_back(DXILOperationDesc(Record)); } - OS << "#ifdef DXIL_OP_ENUM\n"; emitDXILEnums(DXILOps, OS); OS << "#endif\n\n"; - OS << "#ifdef DXIL_OP_INTRINSIC_MAP\n"; emitDXILIntrinsicMap(DXILOps, OS); OS << "#endif\n\n"; - OS << "#ifdef DXIL_OP_OPERATION_TABLE\n"; emitDXILOperationTable(DXILOps, OS); OS << "#endif\n\n"; - - OS << "\n"; } static TableGen::Emitter::Opt X("gen-dxil-operation", EmitDXILOperation, diff --git a/llvm/utils/TableGen/X86DisassemblerTables.cpp b/llvm/utils/TableGen/X86DisassemblerTables.cpp index a48b9cf..f4d282f 100644 --- a/llvm/utils/TableGen/X86DisassemblerTables.cpp +++ b/llvm/utils/TableGen/X86DisassemblerTables.cpp @@ -567,7 +567,9 @@ static inline bool inheritsFrom(InstructionContext child, case IC_EVEX_L2_W_OPSIZE_KZ_B: return false; case IC_EVEX_NF: + return WIG && inheritsFrom(child, IC_EVEX_W_NF); case IC_EVEX_B_NF: + return WIG && inheritsFrom(child, IC_EVEX_W_B_NF); case IC_EVEX_OPSIZE_NF: case IC_EVEX_OPSIZE_B_NF: case IC_EVEX_W_NF: diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py index 4a02a92..5377752 100644 --- a/llvm/utils/UpdateTestChecks/common.py +++ b/llvm/utils/UpdateTestChecks/common.py @@ -388,7 +388,12 @@ def itertests( def should_add_line_to_output( - input_line, prefix_set, skip_global_checks=False, comment_marker=";" + input_line, + prefix_set, + *, + skip_global_checks=False, + skip_same_checks=False, + comment_marker=";", ): # Skip any blank comment lines in the IR. if not skip_global_checks and input_line.strip() == comment_marker: @@ -402,9 +407,14 @@ def should_add_line_to_output( # And skip any CHECK lines. We're building our own. m = CHECK_RE.match(input_line) if m and m.group(1) in prefix_set: + if skip_same_checks and CHECK_SAME_RE.match(input_line): + # The previous CHECK line was removed, so don't leave this dangling + return False if skip_global_checks: + # Skip checks only if they are of global value definitions global_ir_value_re = re.compile(r"(\[\[|@)", flags=(re.M)) - return not global_ir_value_re.search(input_line) + is_global = global_ir_value_re.search(input_line) + return not is_global return False return True @@ -483,6 +493,7 @@ PREFIX_RE = re.compile("^[a-zA-Z0-9_-]+$") CHECK_RE = re.compile( r"^\s*(?://|[;#])\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL|-SAME|-EMPTY)?:" ) +CHECK_SAME_RE = re.compile(r"^\s*(?://|[;#])\s*([^:]+?)(?:-SAME)?:") UTC_ARGS_KEY = "UTC_ARGS:" UTC_ARGS_CMD = re.compile(r".*" + UTC_ARGS_KEY + r"\s*(?P<cmd>.*)\s*$") diff --git a/llvm/utils/gn/secondary/lldb/test/BUILD.gn b/llvm/utils/gn/secondary/lldb/test/BUILD.gn index 06ef738..414ea49 100644 --- a/llvm/utils/gn/secondary/lldb/test/BUILD.gn +++ b/llvm/utils/gn/secondary/lldb/test/BUILD.gn @@ -60,7 +60,8 @@ write_lit_cfg("lit_api_site_cfg") { "LLDB_TEST_COMMON_ARGS=", "LLDB_TEST_USER_ARGS=", "LLDB_ENABLE_PYTHON=0", - "LLDB_HAS_LIBCXX=0", # FIXME: support this (?) + "LLDB_HAS_LIBCXX=False", # FIXME: support this (?) + "LLDB_TEST_USE_VENDOR_PACKAGES=False", "LLDB_LIBS_DIR=", # FIXME: for shared builds only (?) "LLDB_TEST_ARCH=$current_cpu", "LLDB_TEST_COMPILER=" + rebase_path("$root_build_dir/bin/clang"), diff --git a/llvm/utils/update_test_checks.py b/llvm/utils/update_test_checks.py index 06c247c..b5077d7 100755 --- a/llvm/utils/update_test_checks.py +++ b/llvm/utils/update_test_checks.py @@ -235,6 +235,7 @@ def main(): ) else: # "Normal" mode. + dropped_previous_line = False for input_line_info in ti.iterlines(output_lines): input_line = input_line_info.line args = input_line_info.args @@ -282,7 +283,10 @@ def main(): has_checked_pre_function_globals = True if common.should_add_line_to_output( - input_line, prefix_set, not is_in_function + input_line, + prefix_set, + skip_global_checks=not is_in_function, + skip_same_checks=dropped_previous_line, ): # This input line of the function body will go as-is into the output. # Except make leading whitespace uniform: 2 spaces. @@ -290,9 +294,13 @@ def main(): r" ", input_line ) output_lines.append(input_line) + dropped_previous_line = False if input_line.strip() == "}": is_in_function = False continue + else: + # If we are removing a check line, and the next line is CHECK-SAME, it MUST also be removed + dropped_previous_line = True if is_in_function: continue diff --git a/llvm/utils/vim/syntax/mir.vim b/llvm/utils/vim/syntax/mir.vim index 51ac498..024a795 100644 --- a/llvm/utils/vim/syntax/mir.vim +++ b/llvm/utils/vim/syntax/mir.vim @@ -43,6 +43,8 @@ if version >= 508 || !exists("did_c_syn_inits") endif HiLink mirSpecialComment SpecialComment + + delcommand HiLink endif let b:current_syntax = "mir" |